[
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "open-asr-wer",
    "score": 0.0542,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.42%; entry CohereLabs/cohere-transcribe-03-2026; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "open-asr-wer",
    "score": 0.0563,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.63%; entry nvidia/canary-qwen-2.5b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "open-asr-wer",
    "score": 0.0576,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.76%; entry Qwen/Qwen3-ASR-1.7B; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "open-asr-wer",
    "score": 0.0583,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.83%; entry elevenlabs/scribe_v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "open-asr-wer",
    "score": 0.0605,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.05%; entry nvidia/parakeet-tdt-0.6b-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "open-asr-wer",
    "score": 0.0632,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.32%; entry nvidia/parakeet-tdt-0.6b-v3; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "kyutai-stt-2-6b-en",
    "bench": "open-asr-wer",
    "score": 0.064,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.4%; entry kyutai/stt-2.6b-en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "open-asr-wer",
    "score": 0.0662,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.62%; entry mistralai/Voxtral-Small-24B-2507; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "google-chirp-3",
    "bench": "open-asr-wer",
    "score": 0.0663,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.63%; entry google/chirp_3; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "open-asr-wer",
    "score": 0.0691,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.91%; entry speechmatics/enhanced; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "open-asr-wer",
    "score": 0.0705,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.05%; entry mistralai/Voxtral-Mini-3B-2507; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "open-asr-wer",
    "score": 0.071238,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.12375%; entry nvidia/parakeet-rnnt-1.1b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "canary-1b-v2",
    "bench": "open-asr-wer",
    "score": 0.0715,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.15%; entry nvidia/canary-1b-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v3-5",
    "bench": "open-asr-wer",
    "score": 0.0721,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.21%; entry distil-whisper/distil-large-v3.5; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3",
    "bench": "open-asr-wer",
    "score": 0.0744,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.44%; entry openai/whisper-large-v3; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "open-asr-wer",
    "score": 0.0768,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.68%; entry mistralai/Voxtral-Mini-4B-Realtime-2602; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "open-asr-wer",
    "score": 0.0783,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.83%; entry openai/whisper-large-v3-turbo; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "open-asr-longform-wer",
    "score": 0.061967,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 6.196666667 (mean of long-form earnings21 6.48, earnings22 9.99, tedlium 2.12); entry elevenlabs/scribe_v2",
    "source_id": "open-asr"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "open-asr-longform-wer",
    "score": 0.0697,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 6.97 (mean of long-form earnings21 7.9, earnings22 10.75, tedlium 2.26); entry speechmatics/enhanced",
    "source_id": "open-asr"
  },
  {
    "model": "google-chirp-3",
    "bench": "open-asr-longform-wer",
    "score": 0.0962,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.62 (mean of long-form earnings21 11.85, earnings22 14.28, tedlium 2.73); entry google/chirp_3",
    "source_id": "open-asr"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "open-asr-longform-wer",
    "score": 0.078633,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 7.863333333 (mean of long-form earnings21 8.7, earnings22 12.66, tedlium 2.23); entry CohereLabs/cohere-transcribe-03-2026",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "open-asr-longform-wer",
    "score": 0.091033,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.103333333 (mean of long-form earnings21 9.96, earnings22 14.58, tedlium 2.77); entry nvidia/parakeet-tdt-0.6b-v3",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "open-asr-longform-wer",
    "score": 0.087233,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 8.723333333 (mean of long-form earnings21 9.84, earnings22 13.1, tedlium 3.23); entry openai/whisper-large-v3-turbo",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "open-asr-longform-wer",
    "score": 0.090233,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.023333333 (mean of long-form earnings21 10.75, earnings22 13.25, tedlium 3.07); entry nvidia/parakeet-tdt-0.6b-v2",
    "source_id": "open-asr"
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "open-asr-longform-wer",
    "score": 0.085867,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 8.586666667 (mean of long-form earnings21 9.6, earnings22 13.59, tedlium 2.57); entry nvidia/canary-qwen-2.5b",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3",
    "bench": "open-asr-longform-wer",
    "score": 0.086733,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 8.673333333 (mean of long-form earnings21 9.71, earnings22 13.16, tedlium 3.15); entry openai/whisper-large-v3",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v3-5",
    "bench": "open-asr-longform-wer",
    "score": 0.0971,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.71 (mean of long-form earnings21 10.6, earnings22 14.03, tedlium 4.5); entry distil-whisper/distil-large-v3.5",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "open-asr-longform-wer",
    "score": 0.107433,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 10.74333333 (mean of long-form earnings21 12.61, earnings22 15.22, tedlium 4.4); entry nvidia/parakeet-rnnt-1.1b",
    "source_id": "open-asr"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.026731,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 2.673076923 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry elevenlabs/scribe_v2",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.036969,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 3.696923077 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry mistralai/Voxtral-Small-24B-2507",
    "source_id": "open-asr"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0383,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 3.83 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry CohereLabs/cohere-transcribe-03-2026",
    "source_id": "open-asr"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "open-asr-multilingual-wer",
    "score": 0.042938,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.293846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry speechmatics/enhanced",
    "source_id": "open-asr"
  },
  {
    "model": "canary-1b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.045954,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.595384615 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry nvidia/canary-1b-v2",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0481,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.81 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry openai/whisper-large-v3",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "open-asr-multilingual-wer",
    "score": 0.048138,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.813846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry nvidia/parakeet-tdt-0.6b-v3",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.051023,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 5.102307692 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry mistralai/Voxtral-Mini-3B-2507",
    "source_id": "open-asr"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.051146,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 5.114615385 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry Qwen/Qwen3-ASR-1.7B",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "open-asr-multilingual-wer",
    "score": 0.055569,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 5.556923077 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry openai/whisper-large-v3-turbo",
    "source_id": "open-asr"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "open-asr-multilingual-wer",
    "score": 0.063215,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.321538462 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry mistralai/Voxtral-Mini-4B-Realtime-2602",
    "source_id": "open-asr"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-pro",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "MMLU-Pro; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gpqa-diamond",
    "score": 0.762,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "GPQA Diamond; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6",
    "score": 0.558,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-pro",
    "score": 0.825,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "MMLU-Pro; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.817,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "GPQA Diamond; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "gpqa-diamond",
    "score": 0.892,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-08-14",
    "note": "LiveCodeBench v6; HLE judged by GPT-4o; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "hle",
    "score": 0.308,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-08-14",
    "note": "LiveCodeBench v6; HLE judged by GPT-4o; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.852,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "LiveCodeBench v6; SWE-bench with internal agent scaffold; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hle",
    "score": 0.214,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "LiveCodeBench v6; SWE-bench with internal agent scaffold; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.804,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "LiveCodeBench v6; SWE-bench with internal agent scaffold; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.734,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "LiveCodeBench v6; SWE-bench with internal agent scaffold; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmlu-pro",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; HLE w/ CoT (no tools); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "gpqa-diamond",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; HLE w/ CoT (no tools); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hle",
    "score": 0.253,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; HLE w/ CoT (no tools); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "swe-bench-verified",
    "score": 0.72,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; HLE w/ CoT (no tools); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "livecodebench-v6",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; HLE w/ CoT (no tools); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.83,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); HLE marked # on card; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.923,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); HLE marked # on card; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.741,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); HLE marked # on card; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "mmlu-pro",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "aime-2025",
    "score": 0.695,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "livecodebench-v6",
    "score": 0.566,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmlu-pro",
    "score": 0.827,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "aime-2025",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "livecodebench-v6",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-pro",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle",
    "score": 0.287,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "livecodebench-v6",
    "score": 0.836,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "aime-2026",
    "score": 0.913,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-verified",
    "score": 0.764,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "deepseek-r1",
    "bench": "mmlu-pro",
    "score": 0.84,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench Pass@1-COT; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "gpqa-diamond",
    "score": 0.715,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench Pass@1-COT; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "swe-bench-verified",
    "score": 0.492,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench Pass@1-COT; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "mmlu-pro",
    "score": 0.85,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench 2408-2505; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "gpqa-diamond",
    "score": 0.81,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench 2408-2505; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hle",
    "score": 0.177,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench 2408-2505; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "swe-bench-verified",
    "score": 0.576,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench 2408-2505; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2025",
    "score": 0.875,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; LiveCodeBench 2408-2505; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "glm-5-1",
    "bench": "hle",
    "score": 0.31,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "aime-2026",
    "score": 0.953,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "gpqa-diamond",
    "score": 0.862,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.584,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent. Re-filed 2026-10-09 under the split-unspecified id, as the card prints only 'SWE-Bench Pro' (same id as the card's GLM-5 cell).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-2",
    "bench": "hle",
    "score": 0.405,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-2",
    "bench": "aime-2026",
    "score": 0.992,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-2",
    "bench": "gpqa-diamond",
    "score": 0.912,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-2",
    "bench": "swe-bench-pro",
    "score": 0.621,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "minimax-m1",
    "bench": "aime-2025",
    "score": 0.769,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; MiniMax-M1-80K column; LiveCodeBench 24/8-25/5; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1",
    "bench": "gpqa-diamond",
    "score": 0.7,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; MiniMax-M1-80K column; LiveCodeBench 24/8-25/5; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1",
    "bench": "mmlu-pro",
    "score": 0.811,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; MiniMax-M1-80K column; LiveCodeBench 24/8-25/5; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1",
    "bench": "swe-bench-verified",
    "score": 0.56,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; MiniMax-M1-80K column; LiveCodeBench 24/8-25/5; HLE no tools; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "longcat-2-0",
    "bench": "swe-bench-pro",
    "score": 0.595,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "longcat-2-0",
    "bench": "gpqa-diamond",
    "score": 0.889,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "model card; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.696,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.474,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.351,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.74,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.813,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.552,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.613,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.432,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.809,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.66,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "LiveCodeBench v6 (25.02-25.05); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmlu-pro",
    "score": 0.423,
    "source": "https://huggingface.co/Qwen/Qwen3.5-0.8B/blob/2fc06364715b967f1860aea9cf38778875588b17/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "thinking mode; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-0.8b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-pro",
    "score": 0.665,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "thinking mode; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-2b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-pro",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hle",
    "score": 0.243,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "HLE w/ CoT; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "gpqa-diamond",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-verified",
    "score": 0.724,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "livecodebench-v6",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/blob/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hle",
    "score": 0.224,
    "source": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/blob/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "HLE w/ CoT; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.842,
    "source": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/blob/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.692,
    "source": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/blob/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/blob/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.5-35b-a3b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "swe-bench-verified",
    "score": 0.772,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmlu-pro",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "gpqa-diamond",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "hle",
    "score": 0.24,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "livecodebench-v6",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "LiveCodeBench v6; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "aime-2026",
    "score": 0.941,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "mmlu-pro",
    "score": 0.743,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "seed-oss-36b",
    "bench": "mmlu-pro",
    "score": 0.827,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "lab",
    "date": "2025-08-20",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "seed-oss-36b",
    "bench": "aime-2025",
    "score": 0.847,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "lab",
    "date": "2025-08-20",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "seed-oss-36b",
    "bench": "hle",
    "score": 0.101,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "lab",
    "date": "2025-08-20",
    "note": "; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "seed-oss-36b",
    "bench": "livecodebench-v6",
    "score": 0.674,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "lab",
    "date": "2025-08-20",
    "note": "LiveCodeBench v6 (02/2025-05/2025); rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "seed-oss-36b",
    "bench": "swe-bench-verified",
    "score": 0.56,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "lab",
    "date": "2025-08-20",
    "note": "OpenHands; rechecked exact model column and benchmark row in pinned model card; published score converted from percent.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "gpqa-diamond",
    "score": 0.9539141414141414,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id Br3mjs2GrYcCwX9AMwcihU; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01401136318210652.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-micro",
    "bench": "gpqa-diamond",
    "score": 0.2828282828282828,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'granite-4.0-micro'; run id SadaSwnSdMpYRstc8zvgjJ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018237830822136196.",
    "source_id": "epoch-internal"
  },
  {
    "model": "seed-oss-36b",
    "bench": "gpqa-diamond",
    "score": 0.7152777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'seed-oss-36b-instruct'; run id BVat526nF6NUGoVaLCp7DT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02741123270979468.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-32b",
    "bench": "gpqa-diamond",
    "score": 0.6571969696969697,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Qwen3-32B'; run id Ffoxqm7YsZr3FimCGsGAJv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027890331058144285.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.7007575757575758,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-thinking-2507'; run id 2gzNF2d53h2oCKDnojLtSP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028327897613545493.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "gpqa-diamond",
    "score": 0.5561868686868687,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-instruct-2507'; run id X3HfncZtnLKUPoUuRc9YsR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028844434239616544.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "gpqa-diamond",
    "score": 0.3547979797979798,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen2.5-7b-instruct'; run id 6jkd5nJzspGZJtvZynZZTD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02291485990511179.",
    "source_id": "epoch-internal"
  },
  {
    "model": "magistral-small-2509",
    "bench": "gpqa-diamond",
    "score": 0.476010101010101,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'magistral-small-2509'; run id 3UbYVQELDHAcQgu7sTTnRf; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025993462698453398.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gpqa-diamond",
    "score": 0.23926767676767677,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-1B-Instruct'; run id FGEsLTxzZ7fE9aq7DhdqZb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016278803912773237.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7-flash",
    "bench": "gpqa-diamond",
    "score": 0.6054292929292929,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-4.7-flash'; run id GL7htq7XRnbzqpAVK5f862; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02934798760577577.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "gpqa-diamond",
    "score": 0.9577020202020202,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id 3h8STJpG2VzY9LiBUzwotv; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013696812097020311.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.8345959595959596,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.5-35b-a3b'; run id RWjuCuVGvaRd4vDjJJ6aP2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02179018174848238.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "gpqa-diamond",
    "score": 0.4583333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-4b-instruct-2507'; run id VhQBa9AViRt2UKSNpEAyjN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028502193235395257.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-14b",
    "bench": "gpqa-diamond",
    "score": 0.6376262626262627,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-14b'; run id XGK9DjJHHVWr2LyT298ooL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02931953778086544.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-diamond",
    "score": 0.380050505050505,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-1.7b'; run id F3CtjUceBrMi9iYmjULx7J; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02748387280745701.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-2",
    "bench": "gpqa-diamond",
    "score": 0.49053030303030304,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.2-2506'; run id V2CVKXcEdLnEXmppjUEB9M; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027717443261958633.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-4b",
    "bench": "gpqa-diamond",
    "score": 0.23232323232323232,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-3-4b-it'; run id McJyXLHBzx5VjAqXjmrrsb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018690142324718148.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gpqa-diamond",
    "score": 0.1994949494949495,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-3-1b-it'; run id Pc6CyTEYKbcdDUDKzGZnS9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01521201537793133.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-12b",
    "bench": "gpqa-diamond",
    "score": 0.39457070707070707,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-3-12b-it'; run id KfQKr2ibCFMx5xBELoiM3Y; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025254327795483048.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "gpqa-diamond",
    "score": 0.6414141414141414,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-32B'; run id dwyb2yzFewGT6w6oFdVMAc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027534177016461762.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "gpqa-diamond",
    "score": 0.33585858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-1.5B'; run id ESnXA8LqMqhjcsoWQaMLVQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020592418168039042.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "gpqa-diamond",
    "score": 0.0928030303030303,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'deepseek-r1-0528-qwen3-8b'; run id dszRh7ELyCEukWnwYnUxMp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010663485573693416.",
    "source_id": "epoch-internal",
    "invalid": "9.3% on a 4-option test (chance 25%) for a model that scores 61% elsewhere; a reasoning model scoring far below guessing points to unparsed or truncated answers, not ability."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gpqa-diamond",
    "score": 0.26957070707070707,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-8B-Instruct'; run id fxwRTtt8bLV4DNutwtMEox; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018583813167150035.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.7897727272727273,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.5-9B'; run id o6migB5WP27uLr9nUWpCiM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02365568030958479.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8b",
    "bench": "gpqa-diamond",
    "score": 0.5675505050505051,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-8b'; run id jRiWmBxtBAyiXiRBRMY8wX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029327713464985333.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-27b",
    "bench": "gpqa-diamond",
    "score": 0.4772727272727273,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-3-27b-it'; run id jdFH2zZdssYKDVJQPKXvNB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026982120891168943.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "gpqa-diamond",
    "score": 0.4595959595959596,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-oss-20b_high'; run id hYATFPeUnMq7Std2nLUVAJ; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03550702465131341.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "gpqa-diamond",
    "score": 0.9015151515151515,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id Jvt5BNr4ERbE9QGgMcEbmT; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017183231147411448.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "gpqa-diamond",
    "score": 0.9090909090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id dLDBahYBTGRNjbsXA5pyr3; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015851578202142853.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "gpqa-diamond",
    "score": 0.9166666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-18",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id nEJGtXGN588zynPkynZkxz; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015242573629934338.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "gpqa-diamond",
    "score": 0.9482323232323232,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id XyVL3VffJ6UboY97vxpohL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013456525292246259.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "gpqa-diamond",
    "score": 0.88510101010101,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id jTHw5SET8usd7emYPjEDjS; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019100012831823398.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "gpqa-diamond",
    "score": 0.9318181818181818,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id 9YJf2nviwjM6WVhw8qeJta; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015204299687218694.",
    "source_id": "epoch-internal"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "gpqa-diamond",
    "score": 0.8535353535353535,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'nemotron-3-ultra'; run id bpZy8tAbh7UV6A6U7XZtBg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025190921114603887.",
    "source_id": "epoch-internal"
  },
  {
    "model": "minimax-m3",
    "bench": "gpqa-diamond",
    "score": 0.9090909090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'MiniMax-M3'; run id hDESmyGFrbZDomjff3rqzD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020482086775424225.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "gpqa-diamond",
    "score": 0.9185606060606061,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id c5kpB4axzmygRywWqBKFmv; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016123207533687733.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "gpqa-diamond",
    "score": 0.898989898989899,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id CBbeMKRRSeRsXnvSfn7Png; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021469735576055356.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "gpqa-diamond",
    "score": 0.8686868686868687,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_xhigh'; run id gKB2rCD7mHzFN2MCt2YCni; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02406315641682249.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "gpqa-diamond",
    "score": 0.9349747474747475,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id nuKCgZReCBsXKCkNj9bK37; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01570109070867593.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "gpqa-diamond",
    "score": 0.9065656565656566,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-05-05",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_low'; run id 2CdomqtBVz2it7a2Zx2G2t; effort low; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017532658377721708.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "gpqa-diamond",
    "score": 0.9311868686868687,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id B8MaPUe3EYwUZASsNqE7C9; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014937225364194858.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "gpqa-diamond",
    "score": 0.9330808080808081,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id 4bLcFNu5zF6YuZf3qVKoHE; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015363672147634115.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "gpqa-diamond",
    "score": 0.9090909090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id W6mTanyjvXhasFBzHukv4S; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020482086775424225.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "gpqa-diamond",
    "score": 0.8181818181818182,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id evANZQ9oQTdGYbDERw7VrD; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021267123846387202.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "gpqa-diamond",
    "score": 0.8762626262626263,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-11-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_high'; run id CSVtMksEVjBNsGJGjW4GcQ; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01875368281077595.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "gpqa-diamond",
    "score": 0.9160353535353535,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id LWkDtkmd3TNmLzMhfZD5HW; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01728284355494035.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "gpqa-diamond",
    "score": 0.8838383838383839,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id f6ZdMtRpdfskGbrBCzrKSn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022828881775249357.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "gpqa-diamond",
    "score": 0.7960858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id 9Sgsz5XCjscuZkexf7MX9c; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024034764096121958.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "gpqa-diamond",
    "score": 0.8585858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.6-27b'; run id buLf8Ne29xmzMZBRQkvrQG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024825909793343353.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "gpqa-diamond",
    "score": 0.8585858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.5-397b-a17b'; run id gZMFRbVKnBmcnZU4YqqKMN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024825909793343353.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "gpqa-diamond",
    "score": 0.9412878787878788,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id 86SY5JbPB49xXhHFQW6tHu; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014028967942064968.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.8383838383838383,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.6-35b-a3b'; run id EWAyxFsRUvuu4NpmmLqWkn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02622591986362927.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "gpqa-diamond",
    "score": 0.8787878787878788,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.7-plus'; run id URoWjw9Htc6QPwABVcrybw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02325315795194205.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "gpqa-diamond",
    "score": 0.7847,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-04-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id fZB3qGSohAVYzN97mniuUp; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0244.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "gpqa-diamond",
    "score": 0.75,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id jP8EyLtPFsakgnsHe2oJM6; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022767295427113803.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-31b",
    "bench": "gpqa-diamond",
    "score": 0.7575757575757576,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-4-31b-it_minimal'; run id P5L8qbXo42vTvbU3UvTNjk; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03053289223393202.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "gpqa-diamond",
    "score": 0.7323232323232324,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-4-26b-a4b-it_minimal'; run id Usx9KaiXvHZU4qrzgc4sLg; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031544498882702825.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "gpqa-diamond",
    "score": 0.8838383838383839,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id JcQMgXdZKU54u52B8TGQKp; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02282888177524936.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "gpqa-diamond",
    "score": 0.8333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_high'; run id chbh6XstiqtUa5W9zuyfjS; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026552207828215258.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "gpqa-diamond",
    "score": 0.928030303030303,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-05-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id dYYtcwENFRtm9jZTDA2p9C; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016387002835275905.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "gpqa-diamond",
    "score": 0.9444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id BgmrBFsbsaD8rNpebv49Da; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016319950700767385.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "gpqa-diamond",
    "score": 0.8181818181818182,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-flash-lite_high'; run id nE4hmJo3QV6X5jeGWAeAze; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027479603010538825.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "gpqa-diamond",
    "score": 0.803030303030303,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id FNXAt76ZUS3sWhQDFMga7U; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028335609732463334.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "gpqa-diamond",
    "score": 0.7878787878787878,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6_max'; run id SEePRKRcv8oqn5HpD2KFxB; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029126522834586777.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "gpqa-diamond",
    "score": 0.9387626262626263,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id D3mtasTumfvjR4fjmnHWRy; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014782015056089346.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "gpqa-diamond",
    "score": 0.9103535353535354,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-06-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id dqgqP2uEPswfECCVj2Kbr9; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019157125794612026.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "gpqa-diamond",
    "score": 0.8636363636363636,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id Ju7pUkkwYVj6ugi9Ri5Jjp; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02445015597318985.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "gpqa-diamond",
    "score": 0.8585858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id S5pm7ctk9zgPfbyHrbD9AV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024825909793343353.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "gpqa-diamond",
    "score": 0.8825757575757576,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id dfmiZmjAcNkLuAJL4z65eb; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019355903133033585.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "gpqa-diamond",
    "score": 0.9267676767676768,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id TEVNDC9QXBh7PQvX9kHZip; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01686498021450146.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "gpqa-diamond",
    "score": 0.9103535353535354,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id EDFusDaaFkh5n2MqKNxMBD; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017655466909172534.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "gpqa-diamond",
    "score": 0.8617424242424242,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-29",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id 9nPyNrZxwtoT7eS4DZbQF6; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020787836099851995.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "gpqa-diamond",
    "score": 0.933,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-03-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id fFatyce8UvpN7ZivdmrhAy; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "gpqa-diamond",
    "score": 0.6944444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id 2pSDjMSGDh4HsAxkqUuNQP; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027594508998251045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "gpqa-diamond",
    "score": 0.914,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-12-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id Ejo6Fy7XYxUDcCwbejfcYX; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "gpqa-diamond",
    "score": 0.8933080808080808,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id STxFZiSPAPScUuENvXBWLc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01863338951496632.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "gpqa-diamond",
    "score": 0.9343434343434344,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-07-08",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id 9kSgmUd9vh4CBXz3tJhctN; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014340395485738813.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "gpqa-diamond",
    "score": 0.8882575757575758,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id Vw7spGUnJyf3vuzSjTcj2F; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01963840420048217.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "gpqa-diamond",
    "score": 0.9078282828282829,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-05-01",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id LjgJoDosTLkiGVT59smA7q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0171745162866666.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5",
    "bench": "gpqa-diamond",
    "score": 0.8781725888324873,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-5'; run id n4hyz75SVpwPEisKVVjvNg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02336331210092387.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-5",
    "bench": "gpqa-diamond",
    "score": 0.876,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-02-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'fireworks/kimi-k2p5'; run id FmqGUq87N4JcuSbo6Uu3U5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7",
    "bench": "gpqa-diamond",
    "score": 0.8333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-01-29",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'glm-4.7'; run id 6phXzd4aCbnBUSQmBsaywa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02374901666263655.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.8005050505050505,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-12-11",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Qwen3-235B-A22B-Thinking-2507'; run id gu39ndv9QxWP3NRxSUJRaX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026000933515631548.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "gpqa-diamond",
    "score": 0.7575757575757576,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-12-11",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'openai/gpt-oss-120b_high'; run id LC5Rhog6wt2PT84CtJ483h; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027437163443883767.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "gpqa-diamond",
    "score": 0.860479797979798,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-11-24",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id Kwwo3UFcPDk6iLAeSeeRaa; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020686959671393914.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "gpqa-diamond",
    "score": 0.8529040404040404,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-11-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id cqUZ6gcq2HBnShFRWvKjGr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02117010842348584.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "gpqa-diamond",
    "score": 0.8232323232323232,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_59K'; run id XaGuWDxPtqVvyxqdMncKFj; effort 59K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027178752639044908.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "gpqa-diamond",
    "score": 0.7121212121212122,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-haiku-4-5-20251001_32K'; run id GEaAdb9UmoUFbEtBx7WzAR; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03225883512300997.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "gpqa-diamond",
    "score": 0.7070707070707071,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-06-03",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-235b-a22b'; run id agebmxGseTvmvN4AHN8i4m; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027124209393506977.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "gpqa-diamond",
    "score": 0.7632575757575758,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-0528'; run id LiRiryJsGb3h5UuxX8bmJv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024023818843083005.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1",
    "bench": "gpqa-diamond",
    "score": 0.7171717171717171,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1'; run id fFGyQR2mfDufvrSV2rzWyV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03070485693321271.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "gpqa-diamond",
    "score": 0.6584595959595959,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id dXHddeGRX7JRVa5UNfad5C; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027051500965295915.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "gpqa-diamond",
    "score": 0.6691919191919192,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id Gj8pd5gU9r4LqwDK8XZ4mL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02821440093855908.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-scout",
    "bench": "gpqa-diamond",
    "score": 0.5183080808080808,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-4-Scout-17B-16E-Instruct'; run id MsXAXDEKP3xBKtiA7aTBBG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031549020124994755.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-maverick",
    "bench": "gpqa-diamond",
    "score": 0.6698232323232324,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-4-Maverick-17B-128E-Instruct-FP8'; run id LyHy99ubGCqBhoaksQjkiF; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028244823529728173.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "gpqa-diamond",
    "score": 0.6761363636363636,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3-0324'; run id VjfLwKbT6kTprYsMGcz2YT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026836302525030095.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "gpqa-diamond",
    "score": 0.44696969696969696,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-03-10",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-14B'; run id 8UqaLobhmNJLuPGTTpeaCw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02863094678190378.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2",
    "bench": "gpqa-diamond",
    "score": 0.5132575757575758,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-large-2411'; run id n9gpXfUzo8SbWqvrG2q4xM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026887818295509087.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-4",
    "bench": "gpqa-diamond",
    "score": 0.5606060606060606,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'phi-4'; run id RAwuPRwJedy7P2KeQhEmr8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02590274664132679.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "gpqa-diamond",
    "score": 0.46085858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen2.5-32b-instruct'; run id d6JihAFVqUy3u9suNt4P3L; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02595783004579065.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "gpqa-diamond",
    "score": 0.44191919191919193,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-70B-Instruct'; run id MEwMcmSV32JHrsNoAq4BFL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024522863472149384.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "gpqa-diamond",
    "score": 0.5091540404040404,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-405B-Instruct'; run id S5QYXSvQBRSbUbXSnAGbMm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025861992702770648.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "gpqa-diamond",
    "score": 0.4914772727272727,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen2.5-72b-instruct'; run id WtALgW6VSFdihDZWXRm2AZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026947767404178667.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "gpqa-diamond",
    "score": 0.4744318181818182,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.3-70B-Instruct'; run id TxXS78Wg2pSCDbpmeJuzQn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028523027562061026.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3",
    "bench": "gpqa-diamond",
    "score": 0.5653409090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3'; run id Vwo3nMA8g2gGBCyTdKFMQJ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02763990029950893.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "swe-bench-verified",
    "score": 0.787,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id 4AfhcmYVNrw6gM5u8CQsyy; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018656911901656907. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "swe-bench-verified",
    "score": 0.7727272727272727,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-06-18",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id 83hhDhcT9UUrZ2NiUbWSUU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01906835447169652. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "swe-bench-verified",
    "score": 0.7933884297520661,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-06-01",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id dwRzFGTghYEy4HNZajJXoc; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018422420250424888. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-verified",
    "score": 0.7417,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-05-15",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id glm-5.1-swe-bench-frankenstein; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0199. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "swe-bench-verified",
    "score": 0.5785123966942148,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id 8HV4HP9EM5KuLSwzifuLye; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02246855652298021. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "swe-bench-verified",
    "score": 0.7665289256198347,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-05-08",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id bUPtAK7spEwvmiC8pi6afc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01924895909142592. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "swe-bench-verified",
    "score": 0.8347107438016529,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-04-20",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id nCmGWWMBip2s9AVEZNYXDX; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016901169179529604. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "swe-bench-verified",
    "score": 0.768595041322314,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-03-06",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_high'; run id i9HTMbAxcwsrTf2w2FuJ6N; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019189407003691976. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "swe-bench-verified",
    "score": 0.7479338842975206,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-25",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.3-codex_high'; run id 6w5buMbgeEqtRoE55cLZVv; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019756739501473306. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "swe-bench-verified",
    "score": 0.7520661157024794,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-21",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6'; run id KEdgiMmtJKHxgGiS82agYj; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01964818270139032. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "swe-bench-verified",
    "score": 0.7871900826446281,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-18",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6'; run id 2hH2GAPUFaJR7pfVKhhicg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01862353730305288. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "swe-bench-verified",
    "score": 0.6797520661157025,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-18",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_high'; run id YoHtZpMNdsRE6qdj2K4Ucj; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02122974956319103. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-5",
    "bench": "swe-bench-verified",
    "score": 0.737603305785124,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-17",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k2.5'; run id 5wVjUb3yn6bdjPSfPEMuiZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020017832064928274. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5",
    "bench": "swe-bench-verified",
    "score": 0.7207792207792207,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-15",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'glm-5'; run id hKkJEu4ueLiNWsTtrJ78PS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020894141602415234. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "swe-bench-verified",
    "score": 0.5755693581780539,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-13",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id KL724uHFGbiXj4o5PHyLn2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02251275673889269. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "swe-bench-verified",
    "score": 0.737603305785124,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_high'; run id 9owdbpnWJ5pkQQC3GWKP5L; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020017832064928277. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "swe-bench-verified",
    "score": 0.6231884057971014,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_medium'; run id AKt7PcE8LQtoMWVuxwMidk; effort medium; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022072335658124498. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "swe-bench-verified",
    "score": 0.48541666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-08",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id SheUt9cNpkSevy4r6Gsr3u; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022835863746433965. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "swe-bench-verified",
    "score": 0.7355371900826446,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-06",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id C85Bnci9xVzq5jmt8zNnbw; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02006832181710289. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "swe-bench-verified",
    "score": 0.7665289256198347,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-05",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101'; run id mKQKxkpGDrFHBPBYR3mBrB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01924895909142592. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "swe-bench-verified",
    "score": 0.7128099173553719,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-05",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929'; run id Lti64yMEsBWSzgEMFCSVhp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020587234937983246. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "swe-bench-verified",
    "score": 0.6466942148760331,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-01",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_medium'; run id 7nvdQy3AwKtoay2FBvfST5; effort medium; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021749601139671306. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "otis-mock-aime",
    "score": 0.9916666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-17",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.3_xhigh'; run id Wmxkx5m3qaQuwzem6gzZmg; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006154574548966636.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "otis-mock-aime",
    "score": 0.9888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id NeYuqgQyWtFWAHEjQYPm5Z; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008724007409153888.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id mRAASXsUuoJmc2oR5G89zR; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-micro",
    "bench": "otis-mock-aime",
    "score": 0.027777777777777776,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'granite-4.0-micro'; run id gVnaLSupMCiWmiScxfMLx3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014262647723679736.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "otis-mock-aime",
    "score": 0.5444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-35b-a3b'; run id 9WS6kHXT8XFtJWtE2wWLAo; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.061949030495634445.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-32b",
    "bench": "otis-mock-aime",
    "score": 0.6694444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Qwen3-32B'; run id BAi4jidcT3F6reBkSXDJKv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06040106323530329.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "otis-mock-aime",
    "score": 0.7027777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-thinking-2507'; run id 6rZFxNR3GjfhWey6DzEiYz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.060418480757624834.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "otis-mock-aime",
    "score": 0.6222222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-instruct-2507'; run id Eo5cAQJK28CHpKDfCSDYG4; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06509619058885623.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-14b",
    "bench": "otis-mock-aime",
    "score": 0.6638888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-14b'; run id gRRQHECkYhz5UaYsRKDh2F; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06190655406811223.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "otis-mock-aime",
    "score": 0.025,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen2.5-7b-instruct'; run id U8HHNLLLVHJ7sHCfwdiDBX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009401267913629434.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-2",
    "bench": "otis-mock-aime",
    "score": 0.30277777777777776,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.2-2506'; run id HyKzRUkV42yTmGrbYQvZxQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.057774499712758075.",
    "source_id": "epoch-internal"
  },
  {
    "model": "magistral-small-2509",
    "bench": "otis-mock-aime",
    "score": 0.28055555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'magistral-small-2509'; run id cSTYZ3FKp5Mbs3xopn68NK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0556407175551872.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "otis-mock-aime",
    "score": 0.005555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-1B-Instruct'; run id 8qwT2CF9U5zV99PFirnVsp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.003883473863441843.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7-flash",
    "bench": "otis-mock-aime",
    "score": 0.5833333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-4.7-flash'; run id 9ekki5tJhyjmA7ZMLoUxzo; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06564078093085213.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id L7kW8yxFKCjVXhxNnt3VTq; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "seed-oss-36b",
    "bench": "otis-mock-aime",
    "score": 0.675,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'seed-oss-36b-instruct'; run id eJvV6RezECZwh3b4ekg4pm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06324056203789373.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "otis-mock-aime",
    "score": 0.5583333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-4b_none'; run id SAZy8rk2HgyPjA8XZJFUmR; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0616226296489243.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "otis-mock-aime",
    "score": 0.5222222222222223,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-4b-instruct-2507'; run id KH7KfZvFc9kZbqiJFdfYTc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06591537955214949.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "otis-mock-aime",
    "score": 0.08055555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-1.7b_none'; run id bbqokudairnKtbyjBjuWCx; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0281353251405273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "otis-mock-aime",
    "score": 0.8222222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-4-26b-a4b-it_minimal'; run id mstwSH7noJfvhGcd6Lz5kd; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05763774795025092.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-4b",
    "bench": "otis-mock-aime",
    "score": 0.075,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-3-4b-it'; run id KAPGpH2CLMafAt8MnAQJT3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02763853991962833.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-1b",
    "bench": "otis-mock-aime",
    "score": 0.011111111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-3-1b-it'; run id 3CGChL5vNDM6Df5C8Z8owN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005362818328685681.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-12b",
    "bench": "otis-mock-aime",
    "score": 0.16666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-3-12b-it'; run id CAyLyhDHZiMQBZDB3qd8tG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0424174778156649.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "otis-mock-aime",
    "score": 0.5555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-32B'; run id KbUNQf8nnwPa8ixg6AnEwB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.061999963795652153.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "otis-mock-aime",
    "score": 0.5055555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-14B'; run id EceR3NeHzSxacdmAmfRxgT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05984981671790839.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "otis-mock-aime",
    "score": 0.21388888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-1.5B'; run id Gs234P3jzGiLm68KjyzYZL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05279771021800918.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "otis-mock-aime",
    "score": 0.4388888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'deepseek-r1-0528-qwen3-8b'; run id LWbCXNGMXtjQYxyM2uoFMV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0677262181766212.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "otis-mock-aime",
    "score": 0.016666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-8B-Instruct'; run id 6YAzFq89fwWiJ3uJdptCMr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009401267913629434.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "otis-mock-aime",
    "score": 0.42777777777777776,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-9B'; run id 5FVmQUSS4uRd23f6nNgeRe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06533819469304575.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8b",
    "bench": "otis-mock-aime",
    "score": 0.5611111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-8b'; run id gG2B6fLUDBy5jRraxL2pz8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0645035213369854.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-27b",
    "bench": "otis-mock-aime",
    "score": 0.225,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-3-27b-it'; run id DhJ248BX3ZPBHXvkRG4Qsz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.052794388667238824.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "otis-mock-aime",
    "score": 0.5083333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-oss-20b_high'; run id GMngscbSmqoPbGZWNv2Jvw; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06711496995394688.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "otis-mock-aime",
    "score": 0.9388888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id NCikvVWc5SEyMEbCqvVNb2; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021622267380853318.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "otis-mock-aime",
    "score": 0.9111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id goZR5yGxZic9TZ23ezCL3w; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03454754040013868.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "otis-mock-aime",
    "score": 0.9861111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-18",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id 4HVvM9ZWKKPLQPcC8WYYMT; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00990980650017605.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "otis-mock-aime",
    "score": 0.9722222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id kQjrYCaa7yYyMXofeR23Dd; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018586784946171872.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "otis-mock-aime",
    "score": 0.9,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id 53BDFebxiT9KGeP5FRt6bJ; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03964807305493795.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "otis-mock-aime",
    "score": 0.9916666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id 7crLvnfQc987dC6mr3iGbf; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004700633956814717.",
    "source_id": "epoch-internal"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'nemotron-3-ultra'; run id WYTWr3Xk4qxymhMsm7vRQ2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "minimax-m3",
    "bench": "otis-mock-aime",
    "score": 0.7111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'MiniMax-M3'; run id 79ZP6iwoBaF5t9a5VRqPnw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0683294324254051.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "otis-mock-aime",
    "score": 0.8638888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id XjFqou6MyT2XLpkUsCrgxM; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04219779330510574.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "otis-mock-aime",
    "score": 0.9333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id b7Lw9ExxAhVY3hdExoh4MP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03760507165451774.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "otis-mock-aime",
    "score": 0.8888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_xhigh'; run id QGMk6JNCYuUkFMF4zdVCsM; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.047377936967913425.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id 7RjsANKwwsePSstcZeSMhX; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "otis-mock-aime",
    "score": 0.8444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_low'; run id CdmtwZ8NR8YeTXWiLVTRDJ; effort low; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05463890236888291.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "otis-mock-aime",
    "score": 0.9722222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id A33om9MTYMC7xYeLv9X7Ha; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011158359139033182.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "otis-mock-aime",
    "score": 0.9972222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id jUmmCNezoW7jWiK72SVCaP; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.002777777777777778.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "otis-mock-aime",
    "score": 0.9555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id D3YDUZSfEUDsS4frLbEKDu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031067790907534736.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "otis-mock-aime",
    "score": 0.8388888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id 6anLU9CfBBNQkWyJG2GDCa; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04381258550587166.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "otis-mock-aime",
    "score": 0.8861111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-11-13",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_high'; run id mYKKTdV27TRJv9FpUg7rkn; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040479727089927024.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "otis-mock-aime",
    "score": 0.9833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id AhtQ5ggs4pTfLDqHFjueGy; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011650421590325524.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "otis-mock-aime",
    "score": 0.9333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id jYiGAFfUQKJuRStPzU3PX3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03760507165451774.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "otis-mock-aime",
    "score": 0.8166666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id TVsxkJ8ZyfMVmVx88Qq3KN; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.046939139620180814.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "otis-mock-aime",
    "score": 0.9111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.6-27b'; run id 452pB5w5ea6uy92e2q2NMv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04290254662948546.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "otis-mock-aime",
    "score": 0.8888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-397b-a17b'; run id JYuFdyZZSiMvmSZNzMTywb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04737793696791343.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "otis-mock-aime",
    "score": 0.9416666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id ajQFZGMjvyMTH2ULDZhwtz; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030926354359015452.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.6-35b-a3b'; run id 3n5KDcBaKqJh3WTg7qU2rs; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "otis-mock-aime",
    "score": 0.9333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.7-plus'; run id Jvdw6GAXiS6T79bkpbk6Ra; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03760507165451775.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "otis-mock-aime",
    "score": 0.8778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-04-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id jVv9m8DjkdpZaLPdUzAnG9; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0394.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id R82gycJfm5jaCdh32tFkH4; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04286165199944133.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-31b",
    "bench": "otis-mock-aime",
    "score": 0.7333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-4-31b-it_minimal'; run id jFJucVunfLgoQqoFvPVQWP; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06666666666666667.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "otis-mock-aime",
    "score": 0.9555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id nhim2QCRgpir5cZdqrmUz3; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031067790907534736.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "otis-mock-aime",
    "score": 0.9111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id coxYk5vK5cBhQ7iZohMtA5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04290254662948546.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "otis-mock-aime",
    "score": 0.7111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_high'; run id mqsc4vFuKtAHXxbYyX9WwK; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0683294324254051.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "otis-mock-aime",
    "score": 0.9555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-05-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id eYmgjYJfM7Utm46nfoRfZ6; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026696111689702044.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "otis-mock-aime",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-flash-lite_high'; run id NMdaAXyqqPRmWfuxZUfHFF; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06030226891555273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "otis-mock-aime",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id XRutLYUVUePe69APTSTw8S; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06030226891555273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "otis-mock-aime",
    "score": 0.9888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id KxvHAT9SV2dTDn7jBnTk6f; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011111111111111112.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "otis-mock-aime",
    "score": 0.7111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6_max'; run id XnQVH5qSfxVjpzoPwUEmnh; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06832943242540508.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "otis-mock-aime",
    "score": 0.9833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-06-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id 37pJqPpXTE6ia9ikmmGPJ4; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014101901870444154.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id T4y4iGHMqA8tXCdBKr5GjS; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "otis-mock-aime",
    "score": 0.9972222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id eiLEpRUvK48wUKDh4sYkzV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.002777777777777778.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "otis-mock-aime",
    "score": 0.8888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id j2fELMf364qfh2cpaVDL6W; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039171949640122654.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "otis-mock-aime",
    "score": 0.9944444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id aGRKHaq9uSawfYVpC565zU; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0038834738634418433.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "otis-mock-aime",
    "score": 0.9444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id FQ3Suh2vKacz7UNqJybTWk; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020854366486931897.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "otis-mock-aime",
    "score": 0.9138888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-29",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id 3D8aaFBLZwQcnWHYKwvnmq; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03658872186755414.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "otis-mock-aime",
    "score": 0.953,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-03-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id HVsPchr34pWYHNuv5sKYyK; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "otis-mock-aime",
    "score": 0.8111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-31",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id Y9YnaJQWpPFAPhe6jtKMhH; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04626556218905063.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "otis-mock-aime",
    "score": 0.961,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-12-13",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id R393BZjXtkEYju29gs5R8B; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "otis-mock-aime",
    "score": 0.9222222222222223,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-13",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id mM6vdJpgjLY9prnetXzZJS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032829739672413996.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "otis-mock-aime",
    "score": 0.9777777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-08",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id f8Pac6hqib4t7x2s4HCRoT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012743145865697103.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "otis-mock-aime",
    "score": 0.9333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id fzoUFmy77NpEDtXyA6w6ap; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03041173683586219.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "otis-mock-aime",
    "score": 0.9611111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id BTeRdhgt6TFBj6QkGX2j6P; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023777636174410793.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5",
    "bench": "otis-mock-aime",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-5'; run id ccbx7KQY9bTD563vzBXwSg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-5",
    "bench": "otis-mock-aime",
    "score": 0.922,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-02-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'fireworks/kimi-k2p5'; run id mbfZvrSNoXSJqCyD4VKqLe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7",
    "bench": "otis-mock-aime",
    "score": 0.8333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-01-29",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'glm-4.7'; run id GYzmrzMfeSjNjm2J6L9PeT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04767312946227961.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "otis-mock-aime",
    "score": 0.8888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-12-11",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'openai/gpt-oss-120b_high'; run id 3QjP5HPi8tKUzHtNJktfv4; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04463343655613273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-12-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Qwen3-235B-A22B-Thinking-2507'; run id XWpHpVPCaVjcwPpdpXuuGH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "otis-mock-aime",
    "score": 0.8611111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-11-24",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id ikUWdu6HJ2qRgZynfzDCFe; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.042644255740946095.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "otis-mock-aime",
    "score": 0.8472222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-11-16",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id iJp7PgYZa3PiXAV8UWd7MC; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048143196054711346.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "otis-mock-aime",
    "score": 0.7777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_59K'; run id k5PDL6MpCC6P5eLAhodJiS; effort 59K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06267511942419625.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "otis-mock-aime",
    "score": 0.6666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-haiku-4-5-20251001_32K'; run id neZvEzLi6yeL9z5KjGef7H; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07106690545187014.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "otis-mock-aime",
    "score": 0.6638888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-0528'; run id bBV5wCrrZtihrgxEsg9jPD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04911177955575991.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "otis-mock-aime",
    "score": 0.44722222222222224,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id cXbbdQDW2jeYassmRuZYpU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06148017573965347.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "otis-mock-aime",
    "score": 0.38333333333333336,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id VAQtARXUPTpdxGKMekUSbX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06324056203789373.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-maverick",
    "bench": "otis-mock-aime",
    "score": 0.20555555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-4-Maverick-17B-128E-Instruct-FP8'; run id GJEpPThqUxXJBSbvkuXVnw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04948528559541873.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-scout",
    "bench": "otis-mock-aime",
    "score": 0.07777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-4-Scout-17B-16E-Instruct'; run id T3X5DeC47iqCaoRYXu5k8L; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030069597533844094.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "otis-mock-aime",
    "score": 0.37777777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3-0324'; run id VpsMexBEhXkPYJBe7v6Fk9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.062372203800719374.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1",
    "bench": "otis-mock-aime",
    "score": 0.5333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-26",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1'; run id bBY5VZzpDkmTBdSSvu5Ups; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07521014330903548.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "otis-mock-aime",
    "score": 0.05138888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.3-70B-Instruct'; run id ZjsXQpZ7SsejXDo52aetZZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023611111111111107.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "otis-mock-aime",
    "score": 0.07361111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen2.5-32b-instruct'; run id NRCNnCCL5aQKsZf6cKC69Z; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021420595022864277.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2",
    "bench": "otis-mock-aime",
    "score": 0.07777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-large-2411'; run id beRGgJXTJggTi8PPEZNoZ5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024662778950157508.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3",
    "bench": "otis-mock-aime",
    "score": 0.15833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3'; run id 7saSEhfVvXuz4yTcGy5u2y; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0433650088193118.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "otis-mock-aime",
    "score": 0.09722222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-405B-Instruct'; run id 2fQJP3brRApfBdQq7Zy3bm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03218644609162339.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "otis-mock-aime",
    "score": 0.08055555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen2.5-72b-instruct'; run id 67YbVQfJh7sd77KH3zFgTn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026098213018967212.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "otis-mock-aime",
    "score": 0.03611111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-70B-Instruct'; run id X2tQXGELXtEbc2VX6gARd7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01691212350465157.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-4",
    "bench": "otis-mock-aime",
    "score": 0.1375,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'phi-4'; run id 5mmrBSQb7hVSJ9VYtNm46i; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03698083382925635.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "frontiermath",
    "score": 0.7403508771929824,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-18",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.3_max'; run id 6735DShGR4Zu5KZ2U6n4pC; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026016750822385526.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "frontiermath",
    "score": 0.6842105263157895,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id Utgs6DuaQDDdi98ZWH22qL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027582557383453006.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "frontiermath",
    "score": 0.9017543859649123,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id g7dXwZH2tWvqLg3X5idGs2; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017662067789031757.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "frontiermath",
    "score": 0.2771929824561403,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-flash-lite_high'; run id MVZf59jt9uEoWG9w5W9apB; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026560920735701255.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "frontiermath",
    "score": 0.17543859649122806,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.6-35b-a3b'; run id ggG3n6WVPZn7Gu2RNyfQbK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022569134425265196.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "frontiermath",
    "score": 0.3824561403508772,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id jbyVWnEvvFVNNc8AEF6WcQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028838027580159496.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "frontiermath",
    "score": 0.29473684210526313,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.5-397b-a17b'; run id EzMaDdXs4YQTMLgppBGxAb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02705413430767295.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "frontiermath",
    "score": 0.9368421052631579,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id csK2fe5T8p2R7WgoJUu6bZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014434038295343787.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "frontiermath",
    "score": 0.592057761732852,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-19",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id TTuCPacMDpNaCSs3Hpbdjp; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029581952519606228.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "frontiermath",
    "score": 0.8210526315789474,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id YLAk5J3Wd8EruVXee8bAQb; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02274515950332741.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "frontiermath",
    "score": 0.34385964912280703,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.7-plus_none'; run id Tt8tBXHoCEYqGLePWXkM8u; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028185763989072583.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "frontiermath",
    "score": 0.3508771929824561,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.6-27b'; run id VonoEhAsGSJ2FHU33qnhui; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028319255656547093.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "frontiermath",
    "score": 0.3684210526315789,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id Mhwme9odWJMzMjoYkW8r6B; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028623772537150215.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "frontiermath",
    "score": 0.44912280701754387,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id iKtopjTjyCWZqJaEjcc7ZL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029515543245521678.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "frontiermath",
    "score": 0.512280701754386,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_xhigh'; run id jjFRrQvsud3XsgYufr9f8f; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02966059084324674.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "frontiermath",
    "score": 0.3333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id N4HerYebFnJDEU4PEvNNGM; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02797271194322297.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "frontiermath",
    "score": 0.5543859649122806,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id QRDHTGX3rYrVCjVPRarjjt; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029493504108183504.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "frontiermath",
    "score": 0.06666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id 6EQrp2MY4QMYLfTxpioYTe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014801767859562859.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "frontiermath",
    "score": 0.05964912280701754,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id kCG2nzspryMy7LBwWPqwXy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014053607746337793.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "frontiermath",
    "score": 0.36140350877192984,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id dcmRi5JTa27EuwiQTdqr2B; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028506918644975.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "frontiermath",
    "score": 0.4666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id YSR9WCR2SnJYvP7DJwX7YN; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029603535719125714.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "frontiermath",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id JH3PjDrbrQndEcc8fh5Qre; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023735633163877067.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "frontiermath",
    "score": 0.5578947368421052,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id YC2u9wykbYQg4gdkf76j26; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029469977528223976.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "frontiermath",
    "score": 0.6877192982456141,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-25",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id iXvqUzwxAiA3Rg3dwnutFs; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027499133473298815.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "frontiermath",
    "score": 0.6456140350877193,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-19",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id b6yWmeFgsWeEiG9Qzs3NSf; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028383475205435625.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "frontiermath",
    "score": 0.7157894736842105,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id axMdgNDC8UYYLnZYLHZpF5; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026764156649364653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "frontiermath",
    "score": 0.4631578947368421,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id B6bZbqickq8o2fQcgDmzMo; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02958888847874604.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "frontiermath",
    "score": 0.6596491228070176,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id Zb2AKwa942D7dBTTcebAba; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028116467881852295.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "frontiermath",
    "score": 0.3333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id 6KvQvuMMX5zAuteUc6pFcW; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02797271194322297.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "frontiermath",
    "score": 0.7473684210526316,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id ABT89ieSWGnGj4RYMduHiG; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0257841025556124.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "frontiermath",
    "score": 0.5754385964912281,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id mKhVMEicZeGjdvJdTRd7mK; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029329899789943818.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "frontiermath",
    "score": 0.2596491228070175,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_high'; run id 8o2J9uQLKDCTiCFwYdkJRK; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02601675082238553.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "frontiermath",
    "score": 0.5894736842105263,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id nb9LnDeeQyvRCqsbsTANGD; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02919063494391404.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "frontiermath",
    "score": 0.856140350877193,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id ff7GeS7rdperjxSBZ7MHZY; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020824894575343943.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "frontiermath",
    "score": 0.721830985915493,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id bcntamvGsSH6QgETdoaSph; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026636607935112643.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "frontiermath",
    "score": 0.44912280701754387,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id FLDt53waLTeshC7PK9uVy2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029515543245521678.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "frontiermath",
    "score": 0.8596491228070176,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id J6evjTMgHkrun8txnEvzdN; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020611471958164294.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "frontiermath",
    "score": 0.8912280701754386,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id VBpT2WWYKBZtUxG9Z3svGh; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018475392571140503.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "frontiermath",
    "score": 0.5719298245614035,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id djbbEqvBW9qux3pZwcZN2i; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029360921879029937.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "frontiermath",
    "score": 0.656140350877193,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id V6YuGpcu8MCbJNXKoJMA6b; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02818576398907258.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "frontiermath",
    "score": 0.4280701754385965,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id bdsZjrG4ayzP8NAJg5q586; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029360921879029937.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "frontiermath",
    "score": 0.6456140350877193,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id RG2biwnmGybSVTVxxMdRWa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02838347520543562.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "frontiermath",
    "score": 0.24561403508771928,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id FkywQ2KS4AkLPzjSPmhHcS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0255425481026507.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "frontiermath",
    "score": 0.5964912280701754,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview'; run id hVifTUdQZrFEYabgCRu8Qq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02911181956525014.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "frontiermath",
    "score": 0.23859649122807017,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_32K'; run id 7oUkeLfwMK3MtkSNHX6QPu; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02529183228411416.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "frontiermath",
    "score": 0.7859649122807018,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id PaYEQ37A2n6giYBwZGBnXY; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024338000553262393.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "frontiermath",
    "score": 0.8526315789473684,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id 5FGXKvXQwGPtcCX2PNrLR5; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021034091168852485.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "frontiermath",
    "score": 0.674,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id PeZ84GTQdSYQzDxRMWA2Ha; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "frontiermath",
    "score": 0.34385964912280703,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id mS5EwCRZVpvUvJEehPUDZu; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028185763989072587.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "frontiermath",
    "score": 0.6596491228070176,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id kgJ2d6XbKj3dZutDcBYSKU; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028116467881852295.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "frontiermath",
    "score": 0.7017543859649122,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id ZkNQ7GER7Mc9FSPmpUzgKH; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027146911721220194.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "frontiermath",
    "score": 0.5719298245614035,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id BxrNfyMffkVUq4n7U5eK3Q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029360921879029937.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "frontiermath",
    "score": 0.6280701754385964,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id 72DQVwbYiQW5pjhshjBaGU; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028679753751074618.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "frontiermath",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id GVmgtgKMwEqc54f3jA7auP; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023735633163877067.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "frontiermath",
    "score": 0.8701754385964913,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id E28uZiJvrPgKtawbJpoZQ8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019944477920133024.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "simpleqa-verified",
    "score": 0.697,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id gZ4bwozTKtpmZTqnqX3a33; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014539683710535193.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "simpleqa-verified",
    "score": 0.708,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id ExyL3GN8s34tizrXV3VmS3; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014385511563477433.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simpleqa-verified",
    "score": 0.127,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id gYV4CJTjSd7VonfzMmnwcw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010534798620855644.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "simpleqa-verified",
    "score": 0.311,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id avdYKfKUuY6EvfVdxftBE8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014645596385722692.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-1",
    "bench": "simpleqa-verified",
    "score": 0.5778894472361809,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.1'; run id bCdwpRXgZtvQLYC4wA9JXr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015665429072238852.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "simpleqa-verified",
    "score": 0.756,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id PppwnX7EjXx2SeTcFN8Ujn; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013588548437881386.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "simpleqa-verified",
    "score": 0.41,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id TY8R8FV7zCwByEipkqyg2P; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015560917136921659.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-2",
    "bench": "simpleqa-verified",
    "score": 0.6030150753768844,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.2_xhigh'; run id mLLmVVvo6t23MM5yxq7ryW; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015518791563606454.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "simpleqa-verified",
    "score": 0.403,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id bpnzC6CCi3hWuKCfqX4KEY; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015518757419066513.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "simpleqa-verified",
    "score": 0.4044044044044044,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'Qwen3-235B-A22B-Thinking-2507'; run id 4ACEC4x26HWRGiAepHGw43; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015535255725079345.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "simpleqa-verified",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id R2fMFAtRXah864hMifdQGV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014987482264363972.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "simpleqa-verified",
    "score": 0.191,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id eJfbDKqeg9kNXFoxpeV7HG; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012436787112179482.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "simpleqa-verified",
    "score": 0.506,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id mZLWt2JSYvJyGhMvji7CT8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015818160898606836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "simpleqa-verified",
    "score": 0.342,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id WgjFZzojpiv54XqUhvfuUG; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015008706182121802.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7",
    "bench": "simpleqa-verified",
    "score": 0.322,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'glm-4.7'; run id aYau6EG7CCaWVSyVq3vvQc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014782913600996744.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-5",
    "bench": "simpleqa-verified",
    "score": 0.343,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k2.5'; run id RbXXg4JEbTSBnzw3CMSHnD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015019206922357007.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "simpleqa-verified",
    "score": 0.5576730190571715,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id DMjUrG6eZEqwcQ4SqUoViu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015737359397180177.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "simpleqa-verified",
    "score": 0.44144144144144143,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id 4LWNorP8Fy8V8doxirNFcH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015718302238794238.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "simpleqa-verified",
    "score": 0.458,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id bfftd5EzUK9yAekafEofvK; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015763390640483554.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "simpleqa-verified",
    "score": 0.662,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id GJ6YsSuzyRD9YXe6SBwvsu; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014965960710224383.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "simpleqa-verified",
    "score": 0.692,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id 84ARuGyhQsjgh8umRmBvas; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01460648312734278.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "simpleqa-verified",
    "score": 0.5290581162324649,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id 2xfFf6KZAp2gUWYhHGkFgs; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015808394665706597.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "simpleqa-verified",
    "score": 0.33633633633633636,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id TqFmiGvqozt7TSZYmhgF8m; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01495531772489948.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "simpleqa-verified",
    "score": 0.662,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id F83pa7Gfsb4UNVttTnS5Hk; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014965960710224383.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-31b",
    "bench": "simpleqa-verified",
    "score": 0.104,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemma-4-31b-it'; run id D5cbXacPnJ8akq2jAtifEQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009658016218524242.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "simpleqa-verified",
    "score": 0.489,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id dKCWFaoGCWJTnD5CPWQGru; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015815471195292575.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "simpleqa-verified",
    "score": 0.302,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id GVm7khGvtdnQKGfkgAHD8q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014526080235459494.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "simpleqa-verified",
    "score": 0.332,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id Y39ZRp553ULpUDBV2XrBbc; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014899597242811567.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "simpleqa-verified",
    "score": 0.483,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id 7HWW3sjRstKzb9JpcnvLaV; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015810153729833274.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "simpleqa-verified",
    "score": 0.188,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id gaGdPvvfvAw7MjP4HaVQCT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012361586015103758.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "simpleqa-verified",
    "score": 0.371,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id E3325ytkSNYP8ez53VA4fE; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015283736211823095.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "simpleqa-verified",
    "score": 0.494,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id CxHSa2gY7ySiPPGTZEZaRt; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015818160898606833.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "simpleqa-verified",
    "score": 0.117,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id 3QBsxKkbF2R2LEXTd6HDrJ; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010169287802713345.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "simpleqa-verified",
    "score": 0.501,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id gTLurbyKU63i2opsGkJGwh; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015819268290576817.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "simpleqa-verified",
    "score": 0.294,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_high'; run id Z3pjKgAULY9j9wmzQpdqZG; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014414290540008265.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "simpleqa-verified",
    "score": 0.63,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id TijCafTTJUFrJ6jPGvBbgq; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015275252316519465.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "simpleqa-verified",
    "score": 0.48,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_high'; run id QCLnRy69NBH9ip3VYtQneh; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015806639423035177.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "simpleqa-verified",
    "score": 0.307,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_59K'; run id XQEGUQjpXupjdyKoGuSmum; effort 59K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014593284892852741.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "simpleqa-verified",
    "score": 0.451,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id AzCkm282PLhDdLJ8DnAQQc; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015743152379585443.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "simpleqa-verified",
    "score": 0.337,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id LPo74T9L3CawnKX5LLUrSZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014955087918653461.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "simpleqa-verified",
    "score": 0.457,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id CA5LW4phRnHSsZUGXEm8H6; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0157606915901365.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "simpleqa-verified",
    "score": 0.126,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-haiku-4-5-20251001_32K'; run id VhBM9HgJ2RuSVwXriRXMVJ; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010499249222407964.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "simpleqa-verified",
    "score": 0.47,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id 85fhMxLaebWtLoi6oLpTbi; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015790799515836722.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "simpleqa-verified",
    "score": 0.328,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6_max'; run id Zs5fCvNzaz8LhqgwDSAztU; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014853842487270312.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "simpleqa-verified",
    "score": 0.517,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_xhigh'; run id 2c4rvmf5N7QXTugrFLMZ83; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015810153729833274.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "simpleqa-verified",
    "score": 0.53,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id ViC6pxQZUVS3E9M9iEubF7; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015790799515836722.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "simpleqa-verified",
    "score": 0.697,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id UnRUSuZzm2X27TUbpeMPva; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014539683710535193.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "simpleqa-verified",
    "score": 0.432,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id DienatPbgfgn3EzM7Gvzxr; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015672320237336324.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "simpleqa-verified",
    "score": 0.41,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id CeDghdBk8CqtiFSbUzxNH5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015560917136921657.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "simpleqa-verified",
    "score": 0.599,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id Zftz2yk4X5NF28JQzjN3hZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01550610974549837.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "simpleqa-verified",
    "score": 0.707,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_xhigh'; run id SEnvgtunRa5GwAtU2sHa87; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014399942998441292.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "simpleqa-verified",
    "score": 0.735,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id VGeF8Xac4WcSAGgSXVdEBi; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013963164754810065.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "simpleqa-verified",
    "score": 0.349,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id hHnQVatVTbQjM4sfrjw7GH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015080663991563052.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "simpleqa-verified",
    "score": 0.216,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id 8d5RuFTTJKuJ5iJYmPZVJe; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013019735539307761.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "simpleqa-verified",
    "score": 0.117,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id KgUCEfVCyRnY8B5kVChyiD; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010169287802713343.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "epoch-chess",
    "score": 0.38,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-18",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.3_max'; run id iPdvXwCfjkWX8AcpsjR4sZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04878317312145634.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "epoch-chess",
    "score": 0.61,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id BCCBjMXW674jF57sP4jf2h; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04902071300001973.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "epoch-chess",
    "score": 0.47,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id EEGf5SDiF8jXB6NX7eR6Sa; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05016135580465918.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-micro",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'granite-4.0-micro'; run id aExYjnML9LdLYE6fcpdFhe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-35b-a3b'; run id KxxB8kDa9obqrPuec4PEiv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "epoch-chess",
    "score": 0.08,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-thinking-2507'; run id AoYLH4gxsYmhLYi9gyduiM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027265992434429075.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "epoch-chess",
    "score": 0.02,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-30b-a3b-instruct-2507'; run id kHUvo7vG9JDf7fQvWpeP7s; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014070529413628968.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-14b",
    "bench": "epoch-chess",
    "score": 0.04,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-14b'; run id doo2bL7TEHLCNNMwf7NbYN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019694638556693237.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen2.5-7b-instruct'; run id 6Unb9YkKYikGQhXhVgAStG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen2.5-32b-instruct'; run id 2mepC8oUjPQRjq4XcBMpbZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-2",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.2-2506'; run id apM7SYd9nAMf7GtQj94rS2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "magistral-small-2509",
    "bench": "epoch-chess",
    "score": 0.03,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'magistral-small-2509'; run id ZS6cAYkdeNAeRNz7AWQ4LR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01714466079977653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-1B-Instruct'; run id FNMFJuzKKuNsWx9gQoLerW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "epoch-chess",
    "score": 0.72,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id TAGZYToqgQmFZr7uYL4rSF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04512608598542129.",
    "source_id": "epoch-internal"
  },
  {
    "model": "seed-oss-36b",
    "bench": "epoch-chess",
    "score": 0.13,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'seed-oss-36b-instruct'; run id bggkB6SQyMaVNvB2Txr9Nd; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03379976689896311.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-2b_none'; run id QeVP4X52JBDZBdeefZjEmb; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "epoch-chess",
    "score": 0.04,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-4b-instruct-2507'; run id JcgBDysccQrAnWQaUmoehs; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019694638556693237.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-32b",
    "bench": "epoch-chess",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Qwen3-32B'; run id MYGuFuvkWEwLC7US5UR363; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-1.7b_none'; run id 3UzTUnn6wx79jU3XoP7VFA; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-4",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'phi-4'; run id fubPj2m5aKLCeXZAwb3xeD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7-flash",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-4.7-flash'; run id KKzu5PGJQTEtfgMTqe7pnp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-4b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-3-4b-it'; run id QyJe5cEG4rYmwvZv9b3oyG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-1b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-3-1b-it'; run id UbTx6Qbh6T2uXoQbpGV8i3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-12b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-3-12b-it'; run id NaNEVAYahYyobdLPgCeNK9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-32B'; run id cUim7Aqc8ifJMM69uXSeSn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-14B'; run id 2AxSH8gojpyyzADeWfuSqu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-1.5B'; run id a5YYZ3NSBGCUuALngGHhfZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "epoch-chess",
    "score": 0.03,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-r1-0528-qwen3-8b'; run id Yd3YzZmLa8D7eiZWo77WSm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01714466079977653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-8B-Instruct'; run id Q8RAfong7NRCvww73BB3AT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "epoch-chess",
    "score": 0.02,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-9B'; run id 4vjzjypxWVWBVdAnwaDQYX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014070529413628966.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8b",
    "bench": "epoch-chess",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-8b'; run id DPwt4tRRKyqEjPE8Jc2Co9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02190429135575903.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-27b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-3-27b-it'; run id dWG8tET6WnzXeT2WjcjWvT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-oss-20b_high'; run id fjgo9DvKNgVAGtitPvgpY7; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "epoch-chess",
    "score": 0.14,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id b3cBsiDi4D7NtisdffQqxV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0348735088019777.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "epoch-chess",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id 9RfS3yHmzmP3azirghceFL; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "epoch-chess",
    "score": 0.47,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-18",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id EyKecrMhk7Yx25e5HGW43i; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05016135580465918.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "epoch-chess",
    "score": 0.47,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id BnqWATgxsPzSF7ojBEMoG3; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05016135580465918.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "epoch-chess",
    "score": 0.18,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id YyEeZDLsvCT5LdNjR9cUrg; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03861229196653691.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "epoch-chess",
    "score": 0.31,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id RiyrdXiqrXT98mKM9nMsmu; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04648231987117317.",
    "source_id": "epoch-internal"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'nemotron-3-ultra'; run id 26q7yB8XHAcmxWgEMFPMHP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032659863237109045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "minimax-m3",
    "bench": "epoch-chess",
    "score": 0.14,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'MiniMax-M3'; run id XwemU6ABotEuG9o5TRLG2r; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0348735088019777.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "epoch-chess",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id G8hwpomZQaMniWH5XUzTRe; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-1",
    "bench": "epoch-chess",
    "score": 0.19,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.1'; run id kmHbjBrwGgEKNr65JrXcGX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039427724440366255.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "epoch-chess",
    "score": 0.24,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_xhigh'; run id oSkuSAA2Q6oWk6pKKiWnhQ; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04292346959909278.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "epoch-chess",
    "score": 0.64,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_promax'; run id cdASCjQh5E38JhWcm8hHZ7; effort promax; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048241815132442176.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "epoch-chess",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_low'; run id EnbN27oUrNyfbxuPwtZey2; effort low; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0440844002276808.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "epoch-chess",
    "score": 0.39,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id 35hRAbkceJM3Dpb2pWXnx9; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04902071300001973.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "epoch-chess",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id VCogkAgwmUncY2doogWKfz; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04760952285695234.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "epoch-chess",
    "score": 0.54,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id CQhcUcXfPPJKuyt57AHZw5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05009082659620331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "epoch-chess",
    "score": 0.19,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id JfkPdo29um2NkxdhT79AK2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039427724440366255.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "epoch-chess",
    "score": 0.32,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_high'; run id 8aknqG8nWu3P5CoBcetqte; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04688261722621506.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "epoch-chess",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id BrGny9ixHxZTMUT4FpVLSv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0238683256575942.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "epoch-chess",
    "score": 0.4,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id BJiQ8aRjMyCGPTBGYfBhyj; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0492365963917331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "epoch-chess",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus'; run id RXsYuuSedkYZ5ubc8swxGp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03775251680686369.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "epoch-chess",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id k2G5DrB2We92UEZnGoyVuL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04408440022768081.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "epoch-chess",
    "score": 0.22,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-27b'; run id Wh3JxFdvxP688EWwE3t2Uk; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04163331998932266.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-397b-a17b'; run id dagNWyu57ksu7gB7Q86eCx; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03265986323710904.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "epoch-chess",
    "score": 0.4,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id aMtSkxyLZPdf6o6f5zLP6A; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0492365963917331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "epoch-chess",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-35b-a3b'; run id 3y52KdaFo7gQZCccCQnNxp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04408440022768081.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "epoch-chess",
    "score": 0.24,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-plus'; run id hFRxCUnXRNMcpQzUy8GYL5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04292346959909278.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "epoch-chess",
    "score": 0.3,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id eJgi8PW5QQsWq7APrDHbz2; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04605661864718382.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "epoch-chess",
    "score": 0.27,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id dPZQeTGY4XnqpvVjxwJxsT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.044619604333847374.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "epoch-chess",
    "score": 0.3,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-04-14",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id 49RKnpqVsfDx6CvHF42Qqq; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0461.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-31b",
    "bench": "epoch-chess",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-4-31b-it_minimal'; run id 4LjmaK4MRrxuDXUiZsvA3o; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "epoch-chess",
    "score": 0.14,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id FF8iU7vP5ecg6PdvDyE7XG; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0348735088019777.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "epoch-chess",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemma-4-26b-a4b-it_minimal'; run id 272CatAfNpHUw44w7oJ2MM; effort minimal; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0238683256575942.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "epoch-chess",
    "score": 0.22,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_high'; run id BSSNShjap29bDUHz4gA9oh; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.041633319989322654.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "epoch-chess",
    "score": 0.5,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-05-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id imPXZhfTfq9Q9G2kq3stdg; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.050251890762960605.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "epoch-chess",
    "score": 0.49,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id 4mzBGLow3HkS8BmAYCzGTR; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05024183937956914.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-flash-lite_high'; run id YFmtuHAcUZeGwPBe9Jbcxw; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_32K'; run id 6qgxJVk8HHLppb7hPqz9Tv; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032659863237109045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "epoch-chess",
    "score": 0.03,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6_max'; run id TDT98jZXHbUMFYTphGEpqC; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01714466079977653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "epoch-chess",
    "score": 0.42,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id BiaKGj9CUTKhhNwW9SM4qt; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04960449637488583.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "epoch-chess",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-05-29",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id bwak5Cq3LBthfoboicGZUo; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04760952285695234.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "epoch-chess",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id hrRSYo5XSFrcs9fADb5Hox; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id P7kyrA6oQ68tVGHyekP7Qj; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032659863237109045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "epoch-chess",
    "score": 0.41,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id bAR85KQmEATQP8j764jpBA; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04943110704237103.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "epoch-chess",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id VYD3GMYKSkMr75q4CMLDDq; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "epoch-chess",
    "score": 0.29,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id PPbjbUncV6qa9jNRKjYrfp; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04560480215720686.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "epoch-chess",
    "score": 0.33,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id Kk4oLoTLyDuqfnbqjPSvgZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04725815626252608.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "epoch-chess",
    "score": 0.08,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-haiku-4-5-20251001_32K'; run id QUuykbQhKapHn3gjLFjTam; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027265992434429075.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "epoch-chess",
    "score": 0.44,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-03-11",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id RjvLecpyS4VVkJPSXhk8Ju; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "epoch-chess",
    "score": 0.37,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id 6DcmBdRZz57U5cusZNBeGW; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048523658709390974.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "epoch-chess",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id bxF3tvvbxyEMfKeVh7FB2c; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "epoch-chess",
    "score": 0.49,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id manual_gpt_5.2_xhigh_chess_puzzles_frankenstein; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "epoch-chess",
    "score": 0.24,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-13",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id DMei2tkTbCqecGNjYCYYoW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04292346959909278.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "epoch-chess",
    "score": 0.36,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id 6e3LDVuZwoZC4Sg7ushoVP; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048241815132442176.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "epoch-chess",
    "score": 0.16,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-06-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id czNLsHc7tBqBWtWaoTSLXh; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.036845294917747066.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "epoch-chess",
    "score": 0.25,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id R9XpD2PaYpPiokjjSWnMYT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04351941398892446.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "epoch-chess",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-05-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id 4MWuvJB4YXGnGz27Y74cqW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0440844002276808.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5",
    "bench": "epoch-chess",
    "score": 0.1,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5'; run id 7czodbcgEF5zN68ppPFzBN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03015113445777636.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-4-7",
    "bench": "epoch-chess",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-01-29",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-4.7'; run id bzuVVvmLtPCgofXqY7mTVR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016329931618554522.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-5",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-01-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'fireworks/kimi-k2p5'; run id f7vGNi73Cs5wSUsBMSzgFu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032659863237109045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-11",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'openai/gpt-oss-120b_high'; run id h529FgFpbSYUfLa8s3CocQ; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030977345909486177.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-11",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Qwen3-235B-A22B-Thinking-2507'; run id 7iH4kjDYrjgRwQdKYgNACT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03328341697069118.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id mQ3UwD6hUioWSp2s2oELY4; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "epoch-ebr",
    "score": 0.7142857142857142,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-5-5_max'; run id iKWWUmcuzRjqisoshKVEpL; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07063046178186345.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "epoch-ebr",
    "score": 0.5333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-6-sol_max'; run id gM5SADcEixdr2P6PkseLeF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04616837959444122.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "epoch-ebr",
    "score": 0.5714285714285714,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-08",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id 8JukffGviNAYHXSdTJvswB; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06023386019368341.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "epoch-ebr",
    "score": 0.45714285714285713,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-05",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id mtHToEb2uzbaekANakjkJZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03563483225498992.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "epoch-ebr",
    "score": 0.4476190476190475,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-04",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id SjeXjsdda4NKRBTrPiJVM8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.041513323271815936.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "epoch-ebr",
    "score": 0.7619047619047619,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id GhVYxWxttDxZ7akiJQPYsE; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.08384198505551908.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "epoch-ebr",
    "score": 0.30476190476190473,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-08-18",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id VvXTUMGEJiQqgvGMVmaL4m; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016187363535215188.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "epoch-ebr",
    "score": 0.2857142857142857,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id 3zda4kRmrZfxiafpk8BtMM; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014197256999998662.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "epoch-ebr",
    "score": 0.3428571428571428,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id RESzhBbTKsxwmsbBNyXCEz; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01982539047110602.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "epoch-ebr",
    "score": 0.395238,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-07-17",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id h6c9FVqkCj88ZD7ohCipuR; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020141.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "epoch-ebr",
    "score": 0.19047619047619047,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-30",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id Ad4geQVekrBqJn5Ka84awK; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "epoch-ebr",
    "score": 0.126984,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-29",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id XSESb7YgzSTRENEhjHxuwc; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "epoch-ebr",
    "score": 0.126984,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-29",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id VneAiVua3Fv2ySMKyyf4Qp; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "epoch-ebr",
    "score": 0.09523809523809523,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-29",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id U3Fqei2qExVeQDCizpSVfx; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "epoch-ebr",
    "score": 0.230159,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-26",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id 3542DFKGpjD6k8AZfGKD7W; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "epoch-ebr",
    "score": 0.09523809523809523,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id YJY5Dm2gEioHP2ExmTadqG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "epoch-ebr",
    "score": 0.253968,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id RNXPxRZ4vMiwbspiqETy8B; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "epoch-ebr",
    "score": 0.047619047619047616,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id 7QDCZBGyG6w6BxE3gw6Gbv; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "epoch-ebr",
    "score": 0.14285714285714285,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_128K'; run id eTTktoMp5sTthdSqVkLTST; effort 128K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "epoch-ebr",
    "score": 0.023809523809523808,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id QwcSB4hQADcMNQg7EXmaGP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "epoch-ebr",
    "score": 0.023809523809523808,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929'; run id cUQEu2ysx4CcVJHi9KtzjQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "epoch-ebr",
    "score": 0.14285714285714285,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview'; run id GYNraPqbBYhqqAyhYzJfQ7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.4634146341463415,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-18",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.3_max'; run id naZno5TPpgMJfv2b5mhvUW; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07884502378168105.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.21951219512195122,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id Lgy52ZT4YpFrUqqKVu2qUs; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06544589202438408.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.8780487804878049,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id T56N3X4KLA3xz7tf5aCMGb; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051739520574625435.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.976,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id f7BCT27LzcAtaLDP2tcrWF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.17073170731707318,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id iCysWvFJevNyw2dbYosU6J; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05949419959829495.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2926829268292683,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-25",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id 4S3ZrJDkwh4gmGrmJQdvzR; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07194088392074452.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2682926829268293,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-19",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id a4uV8HMMBAcHXLVSmZfBgX; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.36585365853658536,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id KptKXK5nE2Ns2WbiaSSnbc; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07615851217559022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.17073170731707318,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id DVD8G8DHNyzkowBzK7JvMt; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05949419959829495.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.3170731707317073,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id bg3JRdd33iWfvRoGSvyXXE; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07357611282438223.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.04878048780487805,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'Inkling_xhigh'; run id jvJPoRYkPH9Sb5MX252oHp; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03405912205797303.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.4634146341463415,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id PBRdMZyz4L4CqPpXyLD2Lk; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07884502378168105.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.24390243902439024,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id WYs83xB6NwrkMNyQJtuMcQ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06789956540036612.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "frontiermath-tier-4-v2",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_high'; run id juNVfKBcYTwaKnk9GRA2zL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.21951219512195122,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id JCMjXU6FRiiLV3xvs9R2Xv; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06544589202438408.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.7317073170731707,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id ji3kMn49Jcr9t4whxcpjj4; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.3902439024390244,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id gZERSAQwxBAevQzBsWfUie; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07712872341874095.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-20",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.17073170731707318,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.20-0309-reasoning'; run id SX5FCWMk3HF83gGsP3WFA6; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05949419959829495.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.8048780487804879,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_promax'; run id cdWwJSH3jVfrKyhUHmT54T; effort promax; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06265967111543964.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.7073170731707317,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id ZUygpkXqg3NjchXXfedAWV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07194088392074452.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.6097560975609756,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id NLiiJJSyA9wmg2FHENFpAq; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07712872341874097.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.24390243902439024,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-07-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id 7vThbrtbczScg2UcsAYXtF; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06789956540036612.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2926829268292683,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id bqiVinHXFcgyFGPstJwXJN; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07194088392074452.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2926829268292683,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-19",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'glm-5.2_max'; run id JebA8dYLVov6H49ZQSLXh5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07194088392074452.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-3",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.14634146341463414,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'grok-4.3_high'; run id 3iRQkf5v7Wu7YuZhxa76Gg; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.055885069450680974.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.34146341463414637,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id FHZCKbtfvpKGXU99vZFnXN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07497768853141171.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.12195121951219512,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id 2WxRbjq5ZWTgiXZPqYUG4R; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051739520574625435.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.12195121951219512,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id SF6FYnnKztPCDWQc4v4EJa; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051739520574625435.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.024390243902439025,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id P5v3FUnVVTbaqaCr43deKi; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024390243902439025.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.0975609756097561,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_xhigh'; run id jtQ6rDiNxjNKkVMEPBE3Ub; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04691557088212523.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro'; run id 2z7kL9DpfvY44T9Wo99FYx; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2682926829268293,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview'; run id eXVWr3RBeSsWy5DAU8ed46; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.024390243902439025,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_32K'; run id d3m9Xip8cof4s77z6LKkEh; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024390243902439022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.49,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id doiQDghqnLzbE8CFijLNQJ; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.08.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.725,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id PKSMspJiJdNThB6qNvaBSM; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07149950690165274.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.317,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id J3NUxsK2GXq9Whkjn7mEVm; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.073.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.04878048780487805,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id TomkQucNryhFGkaUnUGeLR; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03405912205797303.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.04878048780487805,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_32K'; run id azk4theXxAsizmoi2HDD49; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03405912205797303.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.21951219512195122,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id mSS7C84hJyYi4VnJtgzYvW; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06544589202438408.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2682926829268293,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id 2hbbp8z6PhB7c2T33os9q5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.3170731707317073,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id P4LXQ9PyDDziEANMB5jVRd; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07357611282438223.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2564102564102564,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id eVKA2eavdRAbSN6AnzyBxA; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07083413480167725.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2682926829268293,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id cXn5zzns6VFwbTgotDwZjV; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.5609756097560976,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id JiqgqBTfkGx797uowZgJbk; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07846686801046543.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.902,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id ReDAtJGGfRtrEGGWf4Zbsd; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.046.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "math-level-5",
    "score": 0.9784743202416919,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-10-30",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id 5duoiNDccsgaW7UVt6feYX; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.003369422414145465.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "math-level-5",
    "score": 0.9813066465256798,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-10-29",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id 4wN6FKe2db9CLQ3NUFfYge; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0031922231523012308.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "math-level-5",
    "score": 0.9636087945413192,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-10-22",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-haiku-4-5-20251001_32K'; run id Qwci6JAG4DUWqA9eSHThDJ; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00515811348923114.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "math-level-5",
    "score": 0.9773413897280967,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-10-21",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_32K'; run id FqjAH25VNjgwQTbDJFfqAg; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004091286174812221.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "math-level-5",
    "score": 0.9490181268882175,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id TYQuXJw9H8NRjSovr5VoBn; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004848851575113763.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "math-level-5",
    "score": 0.6885699899295065,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-06-03",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen3-235b-a22b'; run id BTDXT8mHQu4je5jysTDuXV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009487067376018314.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "math-level-5",
    "score": 0.9663897280966768,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-0528'; run id 2wd9khVKiBzsAMUquz2R5s; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0042156161735636515.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "math-level-5",
    "score": 0.978285498489426,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id 727ixGKZ7A7G8LmavE2ScL; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0033204985518619605.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "math-level-5",
    "score": 0.9777190332326284,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id 6oW3GceNQznewQJRwLigth; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0033230338273894273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "math-level-5",
    "score": 0.8729229607250756,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id 6ybVweWdTfiCuirnS2BKXD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007298484758774334.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1",
    "bench": "math-level-5",
    "score": 0.8300604229607251,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-2025-04-14'; run id YyJXPVXJjN5EnLfYjfCTrQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008490146044318322.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-maverick",
    "bench": "math-level-5",
    "score": 0.7301737160120846,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-4-Maverick-17B-128E-Instruct-FP8'; run id HkjCosgMFq5QXB2CyTyU7r; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01083573699636281.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-4-scout",
    "bench": "math-level-5",
    "score": 0.6227341389728097,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-4-Scout-17B-16E-Instruct'; run id c6rTyTFCeoHMBzC2QwjxFL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012022405873783983.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "math-level-5",
    "score": 0.75547583081571,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3-0324'; run id YFyn28VJ6m2hfop3KGRkBs; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008603173106577204.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-3-27b",
    "bench": "math-level-5",
    "score": 0.740370090634441,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-13",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemma-3-27b-it'; run id amrtN7UjAtuyEmv5DaGshP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011195857525727288.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "math-level-5",
    "score": 0.8712235649546828,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-11",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Qwen-14B'; run id R8XGt5ScHpmTbD5DnXR59V; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007486217204155208.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2",
    "bench": "math-level-5",
    "score": 0.5028323262839879,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-large-2411'; run id MMqL6fQSwsBKAdBSfxrYpU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011469451623480956.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1",
    "bench": "math-level-5",
    "score": 0.9305135951661632,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1'; run id XVqLs2khGFSNwEbHJTEgRU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006990872160031429.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-4",
    "bench": "math-level-5",
    "score": 0.6493580060422961,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'phi-4'; run id PnvmfAHVxH9oXfcdcUeUZ3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010656871384084826.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "math-level-5",
    "score": 0.5607061933534743,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-30",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen2.5-32b-instruct'; run id Nk4mmCXFH3mEyCYnQPdPx5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01076720798872177.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "math-level-5",
    "score": 0.22875755287009064,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-8B-Instruct'; run id gyxR3YTcmh4UcKZkET5pPX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008519457631995396.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "math-level-5",
    "score": 0.49773413897280966,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-405B-Instruct'; run id ZR7amdUBRr5Cke8Aoz4snr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012185275936569536.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "math-level-5",
    "score": 0.36678625377643503,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-70B-Instruct'; run id 6JiXiQ67iyiC7cFFRiogvs; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010229604859690931.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "math-level-5",
    "score": 0.41597432024169184,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.3-70B-Instruct'; run id DeaDxFfiQtCTdnZPH7Awaf; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011816670264886683.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "math-level-5",
    "score": 0.631703172205438,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen2.5-72b-instruct'; run id ni4DJboYYpdRssVeAuro2V; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01071424933114761.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3",
    "bench": "math-level-5",
    "score": 0.6485083081570997,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-V3'; run id exvQCLCiga6gE7zvQNp2Z6; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010485766934209063.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "epoch-mirrorcode",
    "score": 0.733,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_high'; run id mirrorcode-ml-2l-claude-fable-5-1; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "epoch-mirrorcode",
    "score": 0.467,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_high'; run id mirrorcode-ml-2l-gpt-6-astra; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "epoch-mirrorcode",
    "score": 0.3111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_high'; run id mirrorcode-ml-2l-claude-opus-4-7; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.09036473223803354.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "epoch-mirrorcode",
    "score": 0.08888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id mirrorcode-ml-2l-gemini-3-1-pro-preview; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04561939004190448.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "epoch-mirrorcode",
    "score": 0.1,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_high'; run id mirrorcode-ml-2l-gpt-5-5; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06042179781166438.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "epoch-mirrorcode",
    "score": 0.15555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_high'; run id mirrorcode-ml-2l-gpt-5-4; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0770945044100374.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "epoch-mirrorcode",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_high'; run id mirrorcode-ml-2l-gpt-5-6-sol; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.09229582069908972.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "epoch-mirrorcode",
    "score": 0.638888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_high'; run id mirrorcode-ml-2l-claude-fable-5; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.10404092940368063.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "epoch-mystery-games",
    "score": 0.25,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-18",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'muse-spark-1.3_max'; run id 6Rcc2q3USiUNB6U44bpvGH; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.044640484731156636.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "epoch-mystery-games",
    "score": 0.47,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id 5yyuCCi2Dpfh6xLBMxqFyB; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05016135580465918.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "epoch-mystery-games",
    "score": 0.58,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id dBHxnnTzLGz3nHis5LET6t; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04960449637488583.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3",
    "bench": "epoch-mystery-games",
    "score": 0.33,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.3_max'; run id UUq99Qhq3qizcoJo5vdhuK; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04725815626252608.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "epoch-mystery-games",
    "score": 0.37,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id T6DyawEuhVmDAaDTzDRNNx; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048523658709390974.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "epoch-mystery-games",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id Epg7eNLwwGPXtX3sSAwPf8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "epoch-mystery-games",
    "score": 0.84,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id FA7pM9hKdkgTXmGHptcno7; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.036845294917747066.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "epoch-mystery-games",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-29",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id eCq63CuekohAswYftWXJVj; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0440844002276808.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "epoch-mystery-games",
    "score": 0.18,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-17",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id RA5MRbmWJpFPR29BQxfvZt; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03861229196653691.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "epoch-mystery-games",
    "score": 0.08,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.3-flash_max'; run id TMM3rPrAHcpBG5Pvw9kr4L; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027265992434429072.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "epoch-mystery-games",
    "score": 0.35,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id MFTd4BUbtzbfcCky6d79d4; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.047937248544110224.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "epoch-mystery-games",
    "score": 0.23,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_high'; run id AsFSc8AQu4F46Tzp4xVwYu; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04229525846816508.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "epoch-mystery-games",
    "score": 0.56,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id EatRQnSaUpUwKzdtjYnQHo; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.049888765156985884.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "epoch-mystery-games",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-plus'; run id DmptTSPXx5uM5TYZoV59JT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03775251680686369.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3",
    "bench": "epoch-mystery-games",
    "score": 0.29,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'o3-2025-04-16_high'; run id FmLdgAHVM7MPn3JebBvxTb; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04560480215720687.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o4-mini",
    "bench": "epoch-mystery-games",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'o4-mini-2025-04-16_high'; run id eJbFTXaxZjnW7HNR93u3WH; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02190429135575903.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "epoch-mystery-games",
    "score": 0.58,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id 5kMRXeasRvtghXpDotCdgV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04960449637488583.",
    "source_id": "epoch-internal"
  },
  {
    "model": "minimax-m3",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'MiniMax-M3'; run id GGpgwGRFVATVJdQr8TgRc9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-max",
    "bench": "epoch-mystery-games",
    "score": 0.32,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-max'; run id f7dotY9NheKNKK2zz6bfKZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04688261722621506.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-1",
    "bench": "epoch-mystery-games",
    "score": 0.16,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.1-2025-11-13_medium'; run id 8dvpS2RJKYLwpVh6M7AJkA; effort medium; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03684529491774706.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5",
    "bench": "epoch-mystery-games",
    "score": 0.23,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-2025-08-07_high'; run id NKdPCHXg9sphwKW2UEnmkD; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04229525846816508.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-mini",
    "bench": "epoch-mystery-games",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-mini-2025-08-07_high'; run id THodvrqzCUw7fjpxvKMMWT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "epoch-mystery-games",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-nano-2026-03-17_high'; run id 2e2NePZeU3iB4wvhwCaTci; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-27b_none'; run id dKZDn279PKuc2Kxm68D4r2; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "epoch-mystery-games",
    "score": 0.18,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-397b-a17b_none'; run id D7WDyCr24AVdzr4eKAreJu; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03861229196653691.",
    "source_id": "epoch-internal"
  },
  {
    "model": "glm-5-2",
    "bench": "epoch-mystery-games",
    "score": 0.15,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'glm-5.2_medium'; run id NfN7VwMPjSVqViqrBQPWk2; effort medium; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03588702812826367.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "epoch-mystery-games",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-122b-a10b_none'; run id X5FL4gkJQYjzJ7axz23rfd; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03775251680686369.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-mini-2026-03-17_medium'; run id 4CzMJvWvuiLMCZhe2JGmSb; effort medium; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "epoch-mystery-games",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-plus_none'; run id jGgkDRHbodcaTzQ3Mp6SxS; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03265986323710904.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "epoch-mystery-games",
    "score": 0.22,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-35b-a3b_none'; run id Ua5RK6JRxdyrHRa3hVFQwq; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.041633319989322654.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-nano",
    "bench": "epoch-mystery-games",
    "score": 0.08,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5-nano-2025-08-07_high'; run id aW4ppDktWqoy5qVWMUws6z; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027265992434429075.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "epoch-mystery-games",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-oss-120b_high'; run id GbQrTQyihBUsb9sxbqT8pT; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "epoch-mystery-games",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'nemotron-3-ultra'; run id U8H8juE3Ko83jsR6TRntbh; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-mini-2025-04-14'; run id giKk2ut3JVFVQUvbykxvSu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "epoch-mystery-games",
    "score": 0.09,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'Qwen3-235B-A22B-Thinking-2507'; run id BP3KYD8rYyFg3LviNGtjLL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028762349126466143.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "epoch-mystery-games",
    "score": 0.43,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-19",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-pro-0813_max'; run id c2KVkMCWnpo6BbnMWgdaDH; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0497569851956243.",
    "source_id": "epoch-internal"
  },
  {
    "model": "inkling-small",
    "bench": "epoch-mystery-games",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-15",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'inkling-small_xhigh'; run id nPFA9ufjfVqg3YNmDRqWeC; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023868325657594204.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "epoch-mystery-games",
    "score": 0.37,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id ZavSMMq9jk5s9WSbARZr6X; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048523658709390974.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "epoch-mystery-games",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id 9cXtChGrowgGDwa9AcAecB; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04760952285695234.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "epoch-mystery-games",
    "score": 0.14,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-6'; run id VjfWxdhdNerrh4huSHT8PT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0348735088019777.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "epoch-mystery-games",
    "score": 0.59,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id FfAvNS5H9bMDSBF4JNNiDz; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04943110704237103.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "epoch-mystery-games",
    "score": 0.28,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id VFYhE54autgdYcMJNXicwU; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04512608598542129.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "epoch-mystery-games",
    "score": 0.25,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id 64rokFyL2L8Vd4SZVdayCy; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04351941398892446.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "epoch-mystery-games",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id g4bbCaiFDqw2YYNDAcmn2a; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04760952285695234.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "epoch-mystery-games",
    "score": 0.35,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-5_max'; run id YdcyYXLzqfPUSSgGv4MTED; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04793724854411023.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "epoch-mystery-games",
    "score": 0.32,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash_high'; run id GQKsxFbdBWB546zE8ci49f; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04688261722621506.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "epoch-mystery-games",
    "score": 0.3,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id YBqiCXPjMUSAgAjDUoz5sy; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04605661864718382.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "epoch-mystery-games",
    "score": 0.19,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3.5-flash-lite_low'; run id Mg2yb9Tp3yc2H3ApJX6JBK; effort low; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039427724440366255.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "epoch-mystery-games",
    "score": 0.34,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-v4-flash-0731_max'; run id 4S4GXbkRhLz8wWP97jYG57; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04760952285695233.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "epoch-mystery-games",
    "score": 0.38,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max_xhigh'; run id c2YLifDU4zYCX5Jbi9HLL7; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048783173121456344.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "epoch-mystery-games",
    "score": 0.52,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-31",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id X29V6KG3aVe5Nv5r3tsLHX; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05021167315686782.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "epoch-mystery-games",
    "score": 0.36,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id Q3WpHdurYNvawj6ycXyj6e; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04824181513244218.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "epoch-mystery-games",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-5-20250929_48K'; run id 4oSbA2RKFsfuAfVayAMAww; effort 48K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03775251680686369.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "epoch-mystery-games",
    "score": 0.22,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_48K'; run id k4zGHaXESxFRTRMkyDLZrz; effort 48K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04163331998932266.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-5",
    "bench": "epoch-furniture",
    "score": 0.225,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-24",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'grok-4.5_high'; run id BFVFWpAaax3RDTXT7apuZE; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.053711144147650755.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-6",
    "bench": "epoch-furniture",
    "score": 0.4,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-24",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'grok-4.6_xhigh'; run id GM5e6UBbgeBJ83NJe6R6y9; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06266221885886569.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "epoch-furniture",
    "score": 0.20833333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-24",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'grok-4.7_xhigh'; run id jWTJKkY9kqYnkvZXCMKnwP; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.052199728697406675.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "epoch-furniture",
    "score": 0.5833333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-23",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-6-sol_max'; run id aV7jRQtNYXjDAcUHhvKxAx; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06418399868645064.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "epoch-furniture",
    "score": 0.8333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-5-5_max'; run id 4QrkRM7KEDCQvg4wvfhz7N; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.047040472216649226.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k3",
    "bench": "epoch-furniture",
    "score": 0.3416666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'kimi-k3_max'; run id fMkb7c7AqFoehCqiy2tLR5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.061169934025019475.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-6",
    "bench": "epoch-furniture",
    "score": 0.21666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'kimi-k2.6'; run id eJcV4jnwhxD6tTxNkrn9Xn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.052301098424620544.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "epoch-furniture",
    "score": 0.26666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gemini-3.7-flash_high'; run id bqeNkcnEXJeZ9jgmao393V; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.056331678510110804.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "epoch-furniture",
    "score": 0.23333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gemini-3.6-flash_high'; run id Kkf7BH3sbSqhEDhhPLcmDN; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05376590116477416.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "epoch-furniture",
    "score": 0.31666666666666665,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gemini-3.8-flash_high'; run id Q2QhUc6V3Cu9XS3WZH4tQh; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0593832076297605.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "epoch-furniture",
    "score": 0.7,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-fable-5-1_max'; run id 9LZMBmKxeBgnf5mvgNvBjW; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05724365605193373.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "epoch-furniture",
    "score": 0.26666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gemini-3.1-pro-preview_high'; run id NzNcASjLtNrngbNQKcD3G3; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.056331678510110804.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "epoch-furniture",
    "score": 0.35833333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-fable-5_max'; run id 433Hqw9fVgtRtAKQfareFr; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06185876325418331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5",
    "bench": "epoch-furniture",
    "score": 0.6083333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-5_max'; run id eBTo3dRKwCVuHaRSyN24VA; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06185876325418331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "epoch-furniture",
    "score": 0.425,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-8_max'; run id mTRwnGKK2QWHrovdPrrMbJ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06380695638890652.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "epoch-furniture",
    "score": 0.3333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-7_max'; run id NPkiPtHFjjYvdXoG2LLg7A; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06137164119322156.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "epoch-furniture",
    "score": 0.2833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-6_max'; run id 4Jq9M5CbuWMyxRHHxavMwA; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05866531842313124.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "epoch-furniture",
    "score": 0.2833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-5-20251101_64K'; run id E3gZb5amR7rcoRkqUGeQRG; effort 64K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05866531842313124.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "epoch-furniture",
    "score": 0.425,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-luna_max'; run id UPAoeCG9AhQYyxCGc8C7Eq; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06380695638890652.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "epoch-furniture",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-6-astra_max'; run id BSzNeBCFhFUnZswSqHtAed; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04928872620664329.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "epoch-furniture",
    "score": 0.5666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-sol_max'; run id SqvYScyg9JZfiUdNLX6V9M; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06340911315327727.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "epoch-furniture",
    "score": 0.5416666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.6-terra_max'; run id dxcm8tfHzGWR8sVJ6kUWMF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06432138878020231.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "epoch-furniture",
    "score": 0.44166666666666665,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.5_xhigh'; run id o7rYVQWSXeZPkF2Smwo9h8; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06410142328406845.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2",
    "bench": "epoch-furniture",
    "score": 0.38333333333333336,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-2025-12-11_xhigh'; run id 9te7vbqxpjSaQgHagpaTFg; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06329764084940143.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4",
    "bench": "epoch-furniture",
    "score": 0.375,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-2026-03-05_xhigh'; run id bJ5Pg8JiLzP3Wfoo9M9TxF; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06246467928511352.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-4-31b",
    "bench": "mmlu-pro",
    "score": 0.852,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmlu-pro",
    "score": 0.826,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "mmlu-pro",
    "score": 0.772,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmlu-pro",
    "score": 0.6940000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmlu-pro",
    "score": 0.6,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmlu-pro",
    "score": 0.6759999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMLU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "aime-2026",
    "score": 0.892,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "aime-2026",
    "score": 0.883,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "aime-2026",
    "score": 0.775,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "aime-2026",
    "score": 0.425,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "aime-2026",
    "score": 0.375,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "aime-2026",
    "score": 0.20800000000000002,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'AIME 2026 no tools', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "livecodebench-v6",
    "score": 0.8,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "livecodebench-v6",
    "score": 0.7709999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "livecodebench-v6",
    "score": 0.72,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-v6",
    "score": 0.52,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "livecodebench-v6",
    "score": 0.44,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "livecodebench-v6",
    "score": 0.29100000000000004,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'LiveCodeBench v6', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "scicode",
    "score": 0.43,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "scicode",
    "score": 0.4,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "scicode",
    "score": 0.38,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "scicode",
    "score": 0.24,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "scicode",
    "score": 0.21,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "scicode",
    "score": 0.21,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'SciCode', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "gpqa-diamond",
    "score": 0.843,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "gpqa-diamond",
    "score": 0.823,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "gpqa-diamond",
    "score": 0.7879999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "gpqa-diamond",
    "score": 0.586,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "gpqa-diamond",
    "score": 0.434,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "gpqa-diamond",
    "score": 0.424,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'GPQA Diamond', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "bbeh-micro",
    "score": 0.7440000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "bbeh-micro",
    "score": 0.648,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "bbeh-micro",
    "score": 0.53,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bbeh-micro",
    "score": 0.331,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "bbeh-micro",
    "score": 0.21899999999999997,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "bbeh-micro",
    "score": 0.193,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Big Bench Extra Hard micro avg', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "hle-gemma4-report",
    "score": 0.195,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'HLE', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "hle-gemma4-report",
    "score": 0.087,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'HLE', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "hle-gemma4-report",
    "score": 0.052000000000000005,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'HLE', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "ifbench",
    "score": 0.76,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "ifbench",
    "score": 0.72,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "ifbench",
    "score": 0.74,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifbench",
    "score": 0.44,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifbench",
    "score": 0.38,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "ifbench",
    "score": 0.32,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFBench', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "ifeval-gemma4-report",
    "score": 0.9890000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "ifeval-gemma4-report",
    "score": 0.985,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "ifeval-gemma4-report",
    "score": 0.972,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifeval-gemma4-report",
    "score": 0.9670000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifeval-gemma4-report",
    "score": 0.946,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "ifeval-gemma4-report",
    "score": 0.904,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'IFEval', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Report-specific metric: no assumption about unreported HLE subset or IFEval aggregation.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "mmmlu",
    "score": 0.884,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmmlu",
    "score": 0.863,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "mmmlu",
    "score": 0.8340000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmmlu",
    "score": 0.7659999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmmlu",
    "score": 0.674,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmmlu",
    "score": 0.7070000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MMMLU', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.664,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.441,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.434,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.254,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.191,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.135,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'MRCR v2 8-needle, 128k', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.36,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.14,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.18,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.08,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.03,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.04,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Terminal Bench Hard', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "tau2-airline-gemma4",
    "score": 0.75,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "tau2-airline-gemma4",
    "score": 0.76,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "tau2-airline-gemma4",
    "score": 0.75,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau2-airline-gemma4",
    "score": 0.52,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau2-airline-gemma4",
    "score": 0.31,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "tau2-airline-gemma4",
    "score": 0.39,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – airline', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "tau2-retail-gemma4",
    "score": 0.8640000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "tau2-retail-gemma4",
    "score": 0.855,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "tau2-retail-gemma4",
    "score": 0.7759999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau2-retail-gemma4",
    "score": 0.6709999999999999,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau2-retail-gemma4",
    "score": 0.34600000000000003,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "tau2-retail-gemma4",
    "score": 0.066,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – retail', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "tau2-telecom",
    "score": 0.693,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "tau2-telecom",
    "score": 0.43,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "tau2-telecom",
    "score": 0.544,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau2-telecom",
    "score": 0.184,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau2-telecom",
    "score": 0.19699999999999998,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "tau2-telecom",
    "score": 0.031,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T5",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 5, row 'Tau2 – telecom', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Agent harness as evaluated in this report. Re-filed 2026-10-09 under tau2-telecom: the report cites Barres et al. 2025 for the three original τ² domains, and the telecom tasks, policy and database are byte-identical across tau2-bench releases v0.1.3–v1.0.1 (its run keeps its own level).",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "mmmu-pro",
    "score": 0.769,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmmu-pro",
    "score": 0.738,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "mmmu-pro",
    "score": 0.691,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmmu-pro",
    "score": 0.526,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmmu-pro",
    "score": 0.442,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmmu-pro",
    "score": 0.49700000000000005,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MMMU Pro', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "math-vision",
    "score": 0.856,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "math-vision",
    "score": 0.8240000000000001,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "math-vision",
    "score": 0.797,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "math-vision",
    "score": 0.595,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "math-vision",
    "score": 0.524,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "math-vision",
    "score": 0.46,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MATH-Vision', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "medxpertqa-mm",
    "score": 0.613,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MedXPertQA MM', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "medxpertqa-mm",
    "score": 0.581,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MedXPertQA MM', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "medxpertqa-mm",
    "score": 0.48700000000000004,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MedXPertQA MM', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "medxpertqa-mm",
    "score": 0.287,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MedXPertQA MM', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "medxpertqa-mm",
    "score": 0.235,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'MedXPertQA MM', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "infographicvqa",
    "score": 0.92,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "infographicvqa",
    "score": 0.893,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "infographicvqa",
    "score": 0.884,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "infographicvqa",
    "score": 0.7,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "infographicvqa",
    "score": 0.639,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "infographicvqa",
    "score": 0.706,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'InfographicVQA', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-31b",
    "bench": "omnidocbench-1-5",
    "score": 0.131,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "omnidocbench-1-5",
    "score": 0.149,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-12b",
    "bench": "omnidocbench-1-5",
    "score": 0.164,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "omnidocbench-1-5",
    "score": 0.181,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "omnidocbench-1-5",
    "score": 0.29,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "gemma-3-27b",
    "bench": "omnidocbench-1-5",
    "score": 0.365,
    "source": "https://arxiv.org/html/2607.02770v1#S2.T6",
    "by": "lab",
    "date": "2026-07-02",
    "note": "Gemma Team (Google DeepMind), Gemma 4 Technical Report v1, table 6, row 'OmniDocBench 1.5 ↓ \\\\downarrow', own Gemma model column. Gemma 4 thinking mode; Gemma 3 27B non-thinking. Max supported resolution (1120 vision tokens); Gemma 3 uses Pan & Scan.",
    "source_id": "gemma4-report"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.8199,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Claude Opus 4.5'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.8029999999999999,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'GPT-5.2 (reasoning: xhigh)'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gpt-5",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7992,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'GPT-5 (reasoning: med)'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "nova-2-pro",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7853,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Nova 2 Pro'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gpt-5-1",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7681999999999999,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'GPT-5.1 (reasoning: high)'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "nova-2-omni",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.757,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Nova 2 Omni'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7332,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Claude Sonnet 4.5'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "nova-2-lite",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7243,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Nova 2 Lite'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.6983,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'GPT-5-mini (reasoning: med)'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.5638000000000001,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Gemini Pro 2.5'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.5344,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Claude Haiku 4.5'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.4157,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Gemini Flash 2.5'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "nova-premier",
    "bench": "massive-agents-ast-10k",
    "score": 0.34049999999999997,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Nova Premier', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "nova-micro",
    "bench": "massive-agents-ast-10k",
    "score": 0.0553,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Nova Micro', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "massive-agents-ast-10k",
    "score": 0.2561,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Llama 3.1 405B', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "massive-agents-ast-10k",
    "score": 0.1877,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Llama 3.1 70B', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "massive-agents-ast-10k",
    "score": 0.0909,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Llama 3.1 8B', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "massive-agents-ast-10k",
    "score": 0.0433,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Llama 3.2 3B', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "massive-agents-ast-10k",
    "score": 0.0023,
    "source": "https://aclanthology.org/2025.findings-emnlp.1099.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 2, PDF page 7 (printed 20199), exact row 'Llama 3.2 1B', AST BFCL Prompt 0-Shot 10k Avg. Macro-average across 52 languages, 9,741 samples; greedy decoding via Amazon Bedrock. Research performed by Amazon AGI. One-shot ablation and per-language/component results not separately counted. Paper published November 2025; date is retrieval (exact run date not given). Table visually checked.",
    "source_id": "massive-agents-paper"
  },
  {
    "model": "deepseek-r1",
    "bench": "aider-polyglot",
    "score": 0.569,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'DeepSeek R1'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.71.2.dev; evaluator commit=5650697-dirty; command=aider --model deepseek/deepseek-reasoner; run=2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-01-20",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "aider-polyglot",
    "score": 0.551,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'DeepSeek V3 (0324)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.78.1.dev; evaluator commit=502b863; command=aider --model deepseek/deepseek-chat; run=2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-03-24",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemma-3-27b",
    "bench": "aider-polyglot",
    "score": 0.049,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'gemma-3-27b-it'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.77.1.dev; evaluator commit=fd21f51-dirty; command=aider --model openrouter/google/gemma-3-27b-it; run=2025-03-15-01-21-24--gemma3-27b-or. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-03-15",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4-1",
    "bench": "aider-polyglot",
    "score": 0.524,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'gpt-4.1'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.81.4.dev; evaluator commit=7a87db5-dirty; command=aider --model gpt-4.1; run=2025-04-14-21-05-54--gpt41-diff-exuser. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-04-14",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "aider-polyglot",
    "score": 0.324,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'gpt-4.1-mini'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.81.4.dev; evaluator commit=ffb743e-dirty; command=aider --model gpt-4.1-mini; run=2025-04-14-21-27-53--gpt41mini-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-04-14",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-5",
    "bench": "aider-polyglot",
    "score": 0.88,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'gpt-5 (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.86.2.dev; evaluator commit=32faf82; command=aider --model openai/gpt-5; run=2025-08-23-15-47-21--gpt-5-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-08-23",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aider-polyglot",
    "score": 0.418,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'gpt-oss-120b (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.85.3.dev; evaluator commit=1af0e59; command=aider --model openrouter/openai/gpt-oss-120b --reasoning-effort high; run=2025-08-06-04-54-48--gpt-oss-120b-high-polyglot. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-08-06",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "kimi-k2",
    "bench": "aider-polyglot",
    "score": 0.591,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'Kimi K2'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.85.3.dev; evaluator commit=915ebff-dirty; command=aider --model openrouter/moonshotai/kimi-k2; run=2025-07-17-17-41-54--kimi-k2-diff-or-pricing. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-07-17",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "llama-4-maverick",
    "bench": "aider-polyglot",
    "score": 0.156,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'Llama 4 Maverick'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.81.2.dev; evaluator commit=9445a31; command=aider --model nvidia_nim/meta/llama-4-maverick-17b-128e-instruct; run=2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-04-06",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o3",
    "bench": "aider-polyglot",
    "score": 0.813,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'o3 (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.84.1.dev; evaluator commit=c48fea6; command=aider --model o3 --reasoning-effort high; run=2025-06-25-21-04-24--o3-price-reduction-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-06-25",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o4-mini",
    "bench": "aider-polyglot",
    "score": 0.72,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'o4-mini (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.82.1.dev; evaluator commit=b66901f-dirty; command=aider --model o4-mini; run=2025-04-16-22-01-58--o4-mini-high-diff-exsys. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-04-16",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "aider-polyglot",
    "score": 0.164,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-Coder-32B-Instruct'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=b51768b0; command=aider --model openai/Qwen2.5-Coder-32B-Instruct; run=2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2024-12-26",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aider-polyglot",
    "score": 0.596,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'Qwen3 235B A22B diff, no think, Alibaba API'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.82.4.dev; evaluator commit=91d7fbd-dirty; command=aider --model openai/qwen3-235b-a22b; run=2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-05-09",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "qwen3-32b",
    "bench": "aider-polyglot",
    "score": 0.4,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "note": "Published entry 'Qwen3 32B'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.82.4.dev; evaluator commit=aaacee5-dirty, aeaf259; command=aider --model openrouter/qwen/qwen3-32b; run=2025-05-08-03-20-24--qwen3-32b-default. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "date": "2025-05-08",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "bfcl-v4",
    "score": 0.687,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Claude-Haiku-4-5-20251001 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "bfcl-v4",
    "score": 0.7747,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Claude-Opus-4-5-20251101 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: claude-opus-4-5-20251101.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "bfcl-v4",
    "score": 0.7324,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Claude-Sonnet-4-5-20250929 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: claude-sonnet-4-5-20250929.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "command-a",
    "bench": "bfcl-v4",
    "score": 0.4649,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Command A (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: command-a-03-2025.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "bfcl-v4",
    "score": 0.5624,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemini-2.5-Flash (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gemini-2.5-flash.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "bfcl-v4",
    "score": 0.3687,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemini-2.5-Flash-Lite (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gemini-2.5-flash-lite.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-12b",
    "bench": "bfcl-v4",
    "score": 0.3043,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemma-3-12b-it (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: google/gemma-3-12b-it.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-1b",
    "bench": "bfcl-v4",
    "score": 0.0717,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemma-3-1b-it (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: google/gemma-3-1b-it.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-27b",
    "bench": "bfcl-v4",
    "score": 0.2947,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemma-3-27b-it (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: google/gemma-3-27b-it.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-4b",
    "bench": "bfcl-v4",
    "score": 0.1962,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Gemma-3-4b-it (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: google/gemma-3-4b-it.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "glm-4-6",
    "bench": "bfcl-v4",
    "score": 0.7238,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GLM-4.6 (FC thinking)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: glm-4.6.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-1",
    "bench": "bfcl-v4",
    "score": 0.5396,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-2025-04-14 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-4.1-2025-04-14.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "bfcl-v4",
    "score": 0.5045,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-mini-2025-04-14 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-4.1-mini-2025-04-14.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-5-2",
    "bench": "bfcl-v4",
    "score": 0.5587,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GPT-5.2-2025-12-11 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-5.2-2025-12-11.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-5-mini",
    "bench": "bfcl-v4",
    "score": 0.5546,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GPT-5-mini-2025-08-07 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-5-mini-2025-08-07.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-5-nano",
    "bench": "bfcl-v4",
    "score": 0.5145,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'GPT-5-nano-2025-08-07 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-5-nano-2025-08-07.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "kimi-k2",
    "bench": "bfcl-v4",
    "score": 0.5906,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Moonshotai-Kimi-K2-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: kimi-k2-0905-preview.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bfcl-v4",
    "score": 0.2583,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-8B-Instruct (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: meta-llama/Llama-3.1-8B-Instruct.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "bfcl-v4",
    "score": 0.1,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-Nemotron-Ultra-253B-v1 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: nvidia/llama-3.1-nemotron-ultra-253b-v1.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "invalid": "No function call was ever emitted: 0.00% on every call-producing category and 100% only on Irrelevance Detection, which rewards not calling. The 10% overall is just the irrelevance weight."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bfcl-v4",
    "score": 0.1082,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-3.2-1B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: meta-llama/Llama-3.2-1B-Instruct.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "bfcl-v4",
    "score": 0.2195,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-3.2-3B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: meta-llama/Llama-3.2-3B-Instruct.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bfcl-v4",
    "score": 0.319,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-3.3-70B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: meta-llama/Llama-3.3-70B-Instruct.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-4-maverick",
    "bench": "bfcl-v4",
    "score": 0.3729,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Maverick uses the explicitly named FP8 checkpoint, not a claimed BF16 run. Harness model identifiers: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-4-scout",
    "bench": "bfcl-v4",
    "score": 0.2813,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Llama-4-Scout-17B-16E-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: meta-llama/Llama-4-Scout-17B-16E-Instruct.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-large-2",
    "bench": "bfcl-v4",
    "score": 0.3837,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'mistral-large-2411 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: mistral-large-2411.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-small-3-2",
    "bench": "bfcl-v4",
    "score": 0.3715,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Mistral-small-2506 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: mistral-small-2506.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-2-lite",
    "bench": "bfcl-v4",
    "score": 0.271,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Amazon-Nova-2-Lite-v1:0 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: us.amazon.nova-2-lite-v1:0.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-micro",
    "bench": "bfcl-v4",
    "score": 0.2229,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Amazon-Nova-Micro-v1:0 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: us.amazon.nova-micro-v1:0.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "o3",
    "bench": "bfcl-v4",
    "score": 0.4856,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'o3-2025-04-16 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: o3-2025-04-16.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "o4-mini",
    "bench": "bfcl-v4",
    "score": 0.5324,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'o4-mini-2025-04-16 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: o4-mini-2025-04-16.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "phi-4",
    "bench": "bfcl-v4",
    "score": 0.2879,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Phi-4 (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: microsoft/phi-4.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "bfcl-v4",
    "score": 0.2393,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-0.6B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-0.6B, qwen3-0.6b.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bfcl-v4",
    "score": 0.2841,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-1.7B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-1.7B, qwen3-1.7b.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-14b",
    "bench": "bfcl-v4",
    "score": 0.4103,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-14B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-14B, qwen3-14b.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "bfcl-v4",
    "score": 0.4799,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-235B-A22B-Instruct-2507 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-235B-A22B-Instruct-2507, qwen3-235b-a22b-instruct-2507.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "bfcl-v4",
    "score": 0.4139,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-30B-A3B-Instruct-2507 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-30B-A3B-Instruct-2507, qwen3-30b-a3b-instruct-2507.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-32b",
    "bench": "bfcl-v4",
    "score": 0.4871,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-32B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-32B, qwen3-32b.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "bfcl-v4",
    "score": 0.3568,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-4B-Instruct-2507 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-4B-Instruct-2507.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-8b",
    "bench": "bfcl-v4",
    "score": 0.4257,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Qwen3-8B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: Qwen/Qwen3-8B, qwen3-8b.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-opus-5",
    "bench": "click-korean-kim",
    "score": 0.9584,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1912/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "click-korean-kim",
    "score": 0.9208,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1837/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "click-korean-kim",
    "score": 0.9063,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1808/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1",
    "bench": "click-korean-kim",
    "score": 0.8055,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1'. gpt-4.1-2025-04-14. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1599/1985 correct after excluding 0 FAILED rows from 1985 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "click-korean-kim",
    "score": 0.7209,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-mini'. gpt-4.1-mini-2025-04-14. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1431/1985 correct after excluding 0 FAILED rows from 1985 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-1",
    "bench": "click-korean-kim",
    "score": 0.8981,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.1(medium)'. GPT-5.1 2025-11-13, reasoning_effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1613/1796 correct after excluding 80 FAILED rows from 1876 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-2",
    "bench": "click-korean-kim",
    "score": 0.9092,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.2(medium)'. GPT-5.2 2025-12-11, reasoning_effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1803/1983 correct after excluding 0 FAILED rows from 1983 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "click-korean-kim",
    "score": 0.9153,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1826/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "click-korean-kim",
    "score": 0.955,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1890/1979 correct after excluding 0 FAILED rows from 1979 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "click-korean-kim",
    "score": 0.9224,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1818/1971 correct after excluding 0 FAILED rows from 1971 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-mini",
    "bench": "click-korean-kim",
    "score": 0.8217,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-mini'. GPT-5-mini; source model-version label 2025-08-08, reasoning=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1525/1856 correct after excluding 20 FAILED rows from 1876 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-nano",
    "bench": "click-korean-kim",
    "score": 0.7205,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-nano'. GPT-5-nano; source model-version label 2025-08-08, reasoning=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1330/1846 correct after excluding 30 FAILED rows from 1876 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "click-korean-kim",
    "score": 0.4782,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-8B-Instruct'. meta-llama/Llama-3.1-8B-Instruct. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 954/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "nova-2-lite",
    "bench": "click-korean-kim",
    "score": 0.6722,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Nova 2 Lite(medium)'. us.amazon.nova-2-lite-v1:0, reasoning=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1341/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "nova-2-pro",
    "bench": "click-korean-kim",
    "score": 0.7308,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Nova 2 Pro - Preview(medium)'. Nova 2 Pro Preview; source run label nova-2-pro-medium-2025-12-11; reasoning=medium. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1458/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4",
    "bench": "click-korean-kim",
    "score": 0.5915,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4'. microsoft/phi-4. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1180/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4-mini",
    "bench": "click-korean-kim",
    "score": 0.4281,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4-mini-instruct'. microsoft/Phi-4-mini-instruct; source label Phi-4-mini-instruct. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "cruxeval-zeroeval",
    "score": 0.735,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-405B-Inst@hyperbolic'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=1.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "cruxeval-zeroeval",
    "score": 0.6425,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-70B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "cruxeval-zeroeval",
    "score": 0.3988,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-8B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.62. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "cruxeval-zeroeval",
    "score": 0.6988,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-32B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.38. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "cruxeval-zeroeval",
    "score": 0.7388,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-72B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "cruxeval-zeroeval",
    "score": 0.5275,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-7B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v3",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8855932203389831,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/deepseek-chat-v3_openai_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'deepseek-chat-v3'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=openai; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export. Run dated 2024-12-26: original V3, before the 0324 update; endpoint is not a weight digest.",
    "date": "2024-12-26",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7726271186440677,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/meta-llama--Llama-3.1-70B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'meta-llama--Llama-3.1-70B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6434745762711864,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/meta-llama--Llama-3.1-8B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'meta-llama--Llama-3.1-8B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8248305084745763,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--Qwen2.5-14B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--Qwen2.5-14B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.875677966101695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--Qwen2.5-32B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--Qwen2.5-32B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8827966101694915,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--Qwen2.5-72B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--Qwen2.5-72B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8002542372881356,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--Qwen2.5-7B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--Qwen2.5-7B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "gsm8k-zeroeval",
    "score": 0.9598,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-405B-Inst@hyperbolic'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "gsm8k-zeroeval",
    "score": 0.9416,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-70B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gsm8k-zeroeval",
    "score": 0.84,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-8B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.38. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-opus-5",
    "bench": "haerae-korean-kim",
    "score": 0.9584,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1474/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "haerae-korean-kim",
    "score": 0.92,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1415/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "haerae-korean-kim",
    "score": 0.9187,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1413/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1",
    "bench": "haerae-korean-kim",
    "score": 0.8583,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1'. gpt-4.1-2025-04-14. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1320/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "haerae-korean-kim",
    "score": 0.7893,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-mini'. gpt-4.1-mini-2025-04-14. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1214/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-1",
    "bench": "haerae-korean-kim",
    "score": 0.9065,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.1(medium)'. GPT-5.1 2025-11-13, reasoning_effort=medium. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1358/1498 correct after excluding 40 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-2",
    "bench": "haerae-korean-kim",
    "score": 0.9181,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.2(medium)'. GPT-5.2 2025-12-11, reasoning_effort=medium. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1412/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "haerae-korean-kim",
    "score": 0.9317,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1433/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "haerae-korean-kim",
    "score": 0.9464,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1448/1530 correct after excluding 0 FAILED rows from 1530 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "haerae-korean-kim",
    "score": 0.9285,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; HAERAE, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1428/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-mini",
    "bench": "haerae-korean-kim",
    "score": 0.8435,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-mini'. GPT-5-mini; source model-version label 2025-08-08, reasoning=medium. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1272/1508 correct after excluding 30 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-nano",
    "bench": "haerae-korean-kim",
    "score": 0.786,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-nano'. GPT-5-nano; source model-version label 2025-08-08, reasoning=medium. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1201/1528 correct after excluding 10 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "haerae-korean-kim",
    "score": 0.539,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-8B-Instruct'. meta-llama/Llama-3.1-8B-Instruct. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 829/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "nova-2-lite",
    "bench": "haerae-korean-kim",
    "score": 0.7393,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Nova 2 Lite(medium)'. us.amazon.nova-2-lite-v1:0, reasoning=medium. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1137/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4",
    "bench": "haerae-korean-kim",
    "score": 0.5923,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4'. microsoft/phi-4. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 911/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4-mini",
    "bench": "haerae-korean-kim",
    "score": 0.4447,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4-mini-instruct'. microsoft/Phi-4-mini-instruct; source label Phi-4-mini-instruct. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-opus-5",
    "bench": "kmmlu-hard-kim",
    "score": 0.884,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3628/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "kmmlu-hard-kim",
    "score": 0.7744,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3175/4100 correct after excluding 0 FAILED rows from 4100 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "kmmlu-hard-kim",
    "score": 0.742,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3045/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1",
    "bench": "kmmlu-hard-kim",
    "score": 0.4279,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1'. gpt-4.1-2025-04-14. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1756/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "kmmlu-hard-kim",
    "score": 0.356,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-mini'. gpt-4.1-mini-2025-04-14. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1461/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-1",
    "bench": "kmmlu-hard-kim",
    "score": 0.7276,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.1(medium)'. GPT-5.1 2025-11-13, reasoning_effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2928/4024 correct after excluding 80 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-2",
    "bench": "kmmlu-hard-kim",
    "score": 0.7463,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.2(medium)'. GPT-5.2 2025-12-11, reasoning_effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3045/4080 correct after excluding 0 FAILED rows from 4080 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "kmmlu-hard-kim",
    "score": 0.7576,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3109/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "kmmlu-hard-kim",
    "score": 0.8504,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3463/4072 correct after excluding 0 FAILED rows from 4072 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "kmmlu-hard-kim",
    "score": 0.7786,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 3158/4056 correct after excluding 0 FAILED rows from 4056 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-mini",
    "bench": "kmmlu-hard-kim",
    "score": 0.6061,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-mini'. GPT-5-mini; source model-version label 2025-08-08, reasoning=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2451/4044 correct after excluding 60 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-nano",
    "bench": "kmmlu-hard-kim",
    "score": 0.5172,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5-nano'. GPT-5-nano; source model-version label 2025-08-08, reasoning=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2102/4064 correct after excluding 40 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "kmmlu-hard-kim",
    "score": 0.2403,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-8B-Instruct'. meta-llama/Llama-3.1-8B-Instruct. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 986/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "nova-2-lite",
    "bench": "kmmlu-hard-kim",
    "score": 0.482,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Nova 2 Lite(medium)'. us.amazon.nova-2-lite-v1:0, reasoning=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1902/3946 correct after excluding 0 FAILED rows from 3946 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "nova-2-pro",
    "bench": "kmmlu-hard-kim",
    "score": 0.5407,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Nova 2 Pro - Preview(medium)'. Nova 2 Pro Preview; source run label nova-2-pro-medium-2025-12-11; reasoning=medium. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2219/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4",
    "bench": "kmmlu-hard-kim",
    "score": 0.2424,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4'. microsoft/phi-4. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 995/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-4-mini",
    "bench": "kmmlu-hard-kim",
    "score": 0.2527,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Phi-4-mini-instruct'. microsoft/Phi-4-mini-instruct; source label Phi-4-mini-instruct. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-opus-5",
    "bench": "kmmlu-pro-kim",
    "score": 0.9564,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2699/2822 correct after excluding 0 FAILED rows from 2822 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "kmmlu-pro-kim",
    "score": 0.8583,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2422/2822 correct after excluding 0 FAILED rows from 2822 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "kmmlu-pro-kim",
    "score": 0.792,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2235/2822 correct after excluding 0 FAILED rows from 2822 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "kmmlu-pro-kim",
    "score": 0.8554,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2414/2822 correct after excluding 0 FAILED rows from 2822 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "kmmlu-pro-kim",
    "score": 0.9364,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2590/2766 correct after excluding 0 FAILED rows from 2766 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "kmmlu-pro-kim",
    "score": 0.8444,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KMMLU-Pro, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 2383/2822 correct after excluding 0 FAILED rows from 2822 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-opus-5",
    "bench": "kobalt-700-kim",
    "score": 0.8814,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 617/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "kobalt-700-kim",
    "score": 0.7614,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 533/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "kobalt-700-kim",
    "score": 0.7329,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 513/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "kobalt-700-kim",
    "score": 0.77,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 539/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "kobalt-700-kim",
    "score": 0.8429,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 590/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "kobalt-700-kim",
    "score": 0.7657,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; KoBALT, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 536/700 correct after excluding 0 FAILED rows from 700 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "mmlu-pro",
    "score": 0.6282413563829787,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Meta-Llama-3_1-70B-Instruct_5shots.zip",
    "by": "independent",
    "note": "Published entry 'meta-llama/Meta-Llama-3.1-70B-Instruct'. Author-released 5-shot CoT predictions, 7559/12032 correct; compare stored pred to answer, including missing predictions as incorrect. Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu-pro",
    "score": 0.44190492021276595,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Meta-Llama-3_1-8B-Instruct_5shots.zip",
    "by": "independent",
    "note": "Published entry 'meta-llama/Meta-Llama-3.1-8B-Instruct'. Author-released 5-shot CoT predictions, 5317/12032 correct; compare stored pred to answer, including missing predictions as incorrect. Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8441,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-405B-Inst@hyperbolic'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.15. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8297,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-70B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.26. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6724,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-8B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=4.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8366,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-32B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8557,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-72B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7513,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-7B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-opus-5",
    "bench": "musr-ko-kim",
    "score": 0.8653,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Opus 5 (reasoning=medium)'. global.anthropic.claude-opus-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 649/750 correct after excluding 0 FAILED rows from 750 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "musr-ko-kim",
    "score": 0.816,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'Claude Sonnet 5 (reasoning=medium)'. global.anthropic.claude-sonnet-5, Bedrock, source version 2026-08-04; adaptive thinking; effort=medium. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 612/750 correct after excluding 0 FAILED rows from 750 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "musr-ko-kim",
    "score": 0.812,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'DeepSeek-V4-Flash-0731 (reasoning=high)'. DeepSeek-V4-Flash-0731, self-hosted vLLM, thinking=True, reasoning_effort=high. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 609/750 correct after excluding 0 FAILED rows from 750 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "musr-ko-kim",
    "score": 0.7147,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Luna (reasoning=medium)'. openai.gpt-5.6-luna, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 536/750 correct after excluding 0 FAILED rows from 750 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "musr-ko-kim",
    "score": 0.8531,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Sol (reasoning=medium)'. openai.gpt-5.6-sol, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 633/742 correct after excluding 0 FAILED rows from 742 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "musr-ko-kim",
    "score": 0.8315,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-5.6 Terra (reasoning=medium)'. openai.gpt-5.6-terra, Bedrock, source version 2026-07-13; reasoning_effort=medium. Personal independent run; MuSR-Ko, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 617/742 correct after excluding 0 FAILED rows from 742 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "deepseek-r1",
    "bench": "zebralogic-zeroeval",
    "score": 0.787,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-R1'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Both raw result files were last changed in commit 4149c25325ac3d45fa0c192e446b911e8ac3d524 on 2025-01-21, before V3-0324 and R1-0528; these are the original releases.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v3",
    "bench": "zebralogic-zeroeval",
    "score": 0.421,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v3'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=27.90. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Both raw result files were last changed in commit 4149c25325ac3d45fa0c192e446b911e8ac3d524 on 2025-01-21, before V3-0324 and R1-0528; these are the original releases.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "zebralogic-zeroeval",
    "score": 0.25,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3.1-405B-Inst@hyperbolic'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 16 evaluated items; field=Puzzle Acc; no-answer percentage=6.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "zebralogic-zeroeval",
    "score": 0.249,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-70B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=43.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "zebralogic-zeroeval",
    "score": 0.128,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-8B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=61.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "zebralogic-zeroeval",
    "score": 0.074,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3.2-3B-Instruct@together'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=54.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "zebralogic-zeroeval",
    "score": 0.261,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-32B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=6.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "zebralogic-zeroeval",
    "score": 0.266,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-72B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=11.90. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "zebralogic-zeroeval",
    "score": 0.12,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-7B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=9.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "aime-2024",
    "score": 0.289,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-1.5B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-1.5B",
        "28.9",
        "52.7",
        "83.9",
        "33.8",
        "16.9",
        "954"
      ],
      "column_index": 1,
      "raw_value": "28.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "math-500",
    "score": 0.839,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-1.5B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-1.5B",
        "28.9",
        "52.7",
        "83.9",
        "33.8",
        "16.9",
        "954"
      ],
      "column_index": 3,
      "raw_value": "83.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "gpqa-diamond",
    "score": 0.338,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-1.5B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-1.5B",
        "28.9",
        "52.7",
        "83.9",
        "33.8",
        "16.9",
        "954"
      ],
      "column_index": 4,
      "raw_value": "33.8",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "livecodebench-unversioned",
    "score": 0.169,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-1.5B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-1.5B",
        "28.9",
        "52.7",
        "83.9",
        "33.8",
        "16.9",
        "954"
      ],
      "column_index": 5,
      "raw_value": "16.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2024",
    "score": 0.555,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-7B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-7B",
        "55.5",
        "83.3",
        "92.8",
        "49.1",
        "37.6",
        "1189"
      ],
      "column_index": 1,
      "raw_value": "55.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-500",
    "score": 0.928,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-7B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-7B",
        "55.5",
        "83.3",
        "92.8",
        "49.1",
        "37.6",
        "1189"
      ],
      "column_index": 3,
      "raw_value": "92.8",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "gpqa-diamond",
    "score": 0.491,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-7B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-7B",
        "55.5",
        "83.3",
        "92.8",
        "49.1",
        "37.6",
        "1189"
      ],
      "column_index": 4,
      "raw_value": "49.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "livecodebench-unversioned",
    "score": 0.376,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-7B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-7B",
        "55.5",
        "83.3",
        "92.8",
        "49.1",
        "37.6",
        "1189"
      ],
      "column_index": 5,
      "raw_value": "37.6",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "aime-2024",
    "score": 0.697,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-14B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-14B",
        "69.7",
        "80.0",
        "93.9",
        "59.1",
        "53.1",
        "1481"
      ],
      "column_index": 1,
      "raw_value": "69.7",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "math-500",
    "score": 0.939,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-14B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-14B",
        "69.7",
        "80.0",
        "93.9",
        "59.1",
        "53.1",
        "1481"
      ],
      "column_index": 3,
      "raw_value": "93.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "gpqa-diamond",
    "score": 0.591,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-14B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-14B",
        "69.7",
        "80.0",
        "93.9",
        "59.1",
        "53.1",
        "1481"
      ],
      "column_index": 4,
      "raw_value": "59.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "livecodebench-unversioned",
    "score": 0.531,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-14B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-14B",
        "69.7",
        "80.0",
        "93.9",
        "59.1",
        "53.1",
        "1481"
      ],
      "column_index": 5,
      "raw_value": "53.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "aime-2024",
    "score": 0.726,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-32B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-32B",
        "72.6",
        "83.3",
        "94.3",
        "62.1",
        "57.2",
        "1691"
      ],
      "column_index": 1,
      "raw_value": "72.6",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "math-500",
    "score": 0.943,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-32B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-32B",
        "72.6",
        "83.3",
        "94.3",
        "62.1",
        "57.2",
        "1691"
      ],
      "column_index": 3,
      "raw_value": "94.3",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "gpqa-diamond",
    "score": 0.621,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-32B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-32B",
        "72.6",
        "83.3",
        "94.3",
        "62.1",
        "57.2",
        "1691"
      ],
      "column_index": 4,
      "raw_value": "62.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "livecodebench-unversioned",
    "score": 0.572,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Qwen-32B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Qwen-32B",
        "72.6",
        "83.3",
        "94.3",
        "62.1",
        "57.2",
        "1691"
      ],
      "column_index": 5,
      "raw_value": "57.2",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "aime-2024",
    "score": 0.504,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-8B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-8B",
        "50.4",
        "80.0",
        "89.1",
        "49.0",
        "39.6",
        "1205"
      ],
      "column_index": 1,
      "raw_value": "50.4",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "math-500",
    "score": 0.891,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-8B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-8B",
        "50.4",
        "80.0",
        "89.1",
        "49.0",
        "39.6",
        "1205"
      ],
      "column_index": 3,
      "raw_value": "89.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "gpqa-diamond",
    "score": 0.49,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-8B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-8B",
        "50.4",
        "80.0",
        "89.1",
        "49.0",
        "39.6",
        "1205"
      ],
      "column_index": 4,
      "raw_value": "49.0",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "livecodebench-unversioned",
    "score": 0.396,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-8B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-8B",
        "50.4",
        "80.0",
        "89.1",
        "49.0",
        "39.6",
        "1205"
      ],
      "column_index": 5,
      "raw_value": "39.6",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "aime-2024",
    "score": 0.7,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-70B",
      "column_label": "AIME 2024 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-70B",
        "70.0",
        "86.7",
        "94.5",
        "65.2",
        "57.5",
        "1633"
      ],
      "column_index": 1,
      "raw_value": "70.0",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "math-500",
    "score": 0.945,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-70B",
      "column_label": "MATH-500 pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-70B",
        "70.0",
        "86.7",
        "94.5",
        "65.2",
        "57.5",
        "1633"
      ],
      "column_index": 3,
      "raw_value": "94.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "gpqa-diamond",
    "score": 0.652,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-70B",
      "column_label": "GPQA Diamond pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-70B",
        "70.0",
        "86.7",
        "94.5",
        "65.2",
        "57.5",
        "1633"
      ],
      "column_index": 4,
      "raw_value": "65.2",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "livecodebench-unversioned",
    "score": 0.575,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-owned distilled checkpoint, exact named row. Maximum generation length 32,768; sampling temperature 0.6, top_p 0.95; 64 responses/query estimate pass@1. AIME cons@64 and Codeforces ratings are not imported. LCB window is not specified in this card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "DeepSeek-R1-Distill-Llama-70B",
      "column_label": "LiveCodeBench pass@1",
      "row": [
        "DeepSeek-R1-Distill-Llama-70B",
        "70.0",
        "86.7",
        "94.5",
        "65.2",
        "57.5",
        "1633"
      ],
      "column_index": 5,
      "raw_value": "57.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "aime-2024",
    "score": 0.86,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact distilled 0528 Qwen3 8B checkpoint, not full-size R1-0528. Source table explicitly labels LCB 2408–2505; evaluation generation maximum 64K tokens, temperature 0.6, top_p 0.95, 16 samples for sampled benchmarks. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "DeepSeek-R1-0528-Qwen3-8B",
      "column_label": "AIME 24",
      "row": [
        "DeepSeek-R1-0528-Qwen3-8B",
        "86.0",
        "76.3",
        "61.5",
        "61.1",
        "60.5"
      ],
      "column_index": 1,
      "raw_value": "86.0",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "aime-2025",
    "score": 0.763,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact distilled 0528 Qwen3 8B checkpoint, not full-size R1-0528. Source table explicitly labels LCB 2408–2505; evaluation generation maximum 64K tokens, temperature 0.6, top_p 0.95, 16 samples for sampled benchmarks. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "DeepSeek-R1-0528-Qwen3-8B",
      "column_label": "AIME 25",
      "row": [
        "DeepSeek-R1-0528-Qwen3-8B",
        "86.0",
        "76.3",
        "61.5",
        "61.1",
        "60.5"
      ],
      "column_index": 2,
      "raw_value": "76.3",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "hmmt-feb-2025",
    "score": 0.615,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact distilled 0528 Qwen3 8B checkpoint, not full-size R1-0528. Source table explicitly labels LCB 2408–2505; evaluation generation maximum 64K tokens, temperature 0.6, top_p 0.95, 16 samples for sampled benchmarks. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "DeepSeek-R1-0528-Qwen3-8B",
      "column_label": "HMMT Feb 25",
      "row": [
        "DeepSeek-R1-0528-Qwen3-8B",
        "86.0",
        "76.3",
        "61.5",
        "61.1",
        "60.5"
      ],
      "column_index": 3,
      "raw_value": "61.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "gpqa-diamond",
    "score": 0.611,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact distilled 0528 Qwen3 8B checkpoint, not full-size R1-0528. Source table explicitly labels LCB 2408–2505; evaluation generation maximum 64K tokens, temperature 0.6, top_p 0.95, 16 samples for sampled benchmarks. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "DeepSeek-R1-0528-Qwen3-8B",
      "column_label": "GPQA Diamond",
      "row": [
        "DeepSeek-R1-0528-Qwen3-8B",
        "86.0",
        "76.3",
        "61.5",
        "61.1",
        "60.5"
      ],
      "column_index": 4,
      "raw_value": "61.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.605,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact distilled 0528 Qwen3 8B checkpoint, not full-size R1-0528. Source table explicitly labels LCB 2408–2505; evaluation generation maximum 64K tokens, temperature 0.6, top_p 0.95, 16 samples for sampled benchmarks. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "DeepSeek-R1-0528-Qwen3-8B",
      "column_label": "LiveCodeBench (2408-2505)",
      "row": [
        "DeepSeek-R1-0528-Qwen3-8B",
        "86.0",
        "76.3",
        "61.5",
        "61.1",
        "60.5"
      ],
      "column_index": 5,
      "raw_value": "60.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "phi-4-mini",
    "bench": "bbh",
    "score": 0.704,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: BigBench Hard (0-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "BigBench Hard (0-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "BigBench Hard (0-shot, CoT)",
        "70.4",
        "63.1",
        "55.4",
        "51.2",
        "56.2",
        "72.4",
        "53.3",
        "63.4",
        "55.5",
        "65.7",
        "80.4"
      ],
      "column_index": 1,
      "raw_value": "70.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "mmlu",
    "score": 0.673,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: MMLU (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MMLU (5-shot)",
      "column_label": "Similar size",
      "row": [
        "MMLU (5-shot)",
        "67.3",
        "65.5",
        "61.8",
        "60.8",
        "65.0",
        "72.6",
        "63.0",
        "68.1",
        "65.0",
        "71.3",
        "77.2"
      ],
      "column_index": 1,
      "raw_value": "67.3",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "mmlu-pro",
    "score": 0.528,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: MMLU-Pro (0-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MMLU-Pro (0-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "MMLU-Pro (0-shot, CoT)",
        "52.8",
        "47.4",
        "39.2",
        "35.3",
        "44.7",
        "56.2",
        "36.6",
        "44.0",
        "40.9",
        "50.1",
        "62.8"
      ],
      "column_index": 1,
      "raw_value": "52.8",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "arc-challenge",
    "score": 0.837,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: ARC Challenge (10-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "ARC Challenge (10-shot)",
      "column_label": "Similar size",
      "row": [
        "ARC Challenge (10-shot)",
        "83.7",
        "84.6",
        "76.1",
        "80.3",
        "82.6",
        "90.1",
        "82.7",
        "83.1",
        "79.4",
        "89.8",
        "93.5"
      ],
      "column_index": 1,
      "raw_value": "83.7",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "boolq",
    "score": 0.812,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: BoolQ (2-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "BoolQ (2-shot)",
      "column_label": "Similar size",
      "row": [
        "BoolQ (2-shot)",
        "81.2",
        "77.7",
        "71.4",
        "79.4",
        "65.4",
        "80.0",
        "80.5",
        "82.8",
        "79.3",
        "85.7",
        "88.7"
      ],
      "column_index": 1,
      "raw_value": "81.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "gpqa-unspecified",
    "score": 0.252,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: GPQA (0-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "GPQA (0-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "GPQA (0-shot, CoT)",
        "25.2",
        "26.6",
        "24.3",
        "24.4",
        "23.4",
        "30.6",
        "26.3",
        "26.3",
        "29.9",
        "39.1",
        "41.1"
      ],
      "column_index": 1,
      "raw_value": "25.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "hellaswag",
    "score": 0.691,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: HellaSwag (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HellaSwag (5-shot)",
      "column_label": "Similar size",
      "row": [
        "HellaSwag (5-shot)",
        "69.1",
        "72.2",
        "77.2",
        "74.6",
        "74.6",
        "80.0",
        "73.5",
        "72.8",
        "80.9",
        "87.1",
        "88.7"
      ],
      "column_index": 1,
      "raw_value": "69.1",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "openbookqa",
    "score": 0.792,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: OpenBookQA (10-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "OpenBookQA (10-shot)",
      "column_label": "Similar size",
      "row": [
        "OpenBookQA (10-shot)",
        "79.2",
        "81.2",
        "72.6",
        "79.8",
        "79.3",
        "82.6",
        "80.2",
        "84.8",
        "79.8",
        "90.0",
        "90.0"
      ],
      "column_index": 1,
      "raw_value": "79.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "piqa",
    "score": 0.776,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: PIQA (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "PIQA (5-shot)",
      "column_label": "Similar size",
      "row": [
        "PIQA (5-shot)",
        "77.6",
        "78.2",
        "68.2",
        "73.2",
        "72.6",
        "76.2",
        "81.2",
        "83.2",
        "78.3",
        "83.7",
        "88.7"
      ],
      "column_index": 1,
      "raw_value": "77.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "social-iqa",
    "score": 0.725,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: Social IQA (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Social IQA (5-shot)",
      "column_label": "Similar size",
      "row": [
        "Social IQA (5-shot)",
        "72.5",
        "75.1",
        "68.3",
        "73.9",
        "75.3",
        "75.3",
        "77.6",
        "71.8",
        "73.4",
        "74.7",
        "82.9"
      ],
      "column_index": 1,
      "raw_value": "72.5",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "truthfulqa-mc2",
    "score": 0.664,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: TruthfulQA (MC2) (10-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "TruthfulQA (MC2) (10-shot)",
      "column_label": "Similar size",
      "row": [
        "TruthfulQA (MC2) (10-shot)",
        "66.4",
        "65.2",
        "59.2",
        "62.9",
        "64.3",
        "69.4",
        "63.0",
        "69.2",
        "64.1",
        "76.6",
        "78.2"
      ],
      "column_index": 1,
      "raw_value": "66.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "winogrande",
    "score": 0.67,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: Winogrande (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Winogrande (5-shot)",
      "column_label": "Similar size",
      "row": [
        "Winogrande (5-shot)",
        "67.0",
        "72.2",
        "53.2",
        "59.8",
        "63.3",
        "71.1",
        "63.1",
        "64.7",
        "65.4",
        "74.0",
        "76.9"
      ],
      "column_index": 1,
      "raw_value": "67.0",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "multilingual-mmlu",
    "score": 0.493,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: Multilingual MMLU (5-shot). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multilingual MMLU (5-shot)",
      "column_label": "Similar size",
      "row": [
        "Multilingual MMLU (5-shot)",
        "49.3",
        "51.8",
        "48.1",
        "46.4",
        "55.9",
        "64.4",
        "53.7",
        "56.2",
        "54.5",
        "63.8",
        "72.9"
      ],
      "column_index": 1,
      "raw_value": "49.3",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "mgsm",
    "score": 0.639,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: MGSM (0-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MGSM (0-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "MGSM (0-shot, CoT)",
        "63.9",
        "49.6",
        "44.6",
        "44.6",
        "53.5",
        "64.5",
        "56.7",
        "56.7",
        "58.6",
        "75.1",
        "81.7"
      ],
      "column_index": 1,
      "raw_value": "63.9",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "gsm8k",
    "score": 0.886,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: GSM8K (8-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "GSM8K (8-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "GSM8K (8-shot, CoT)",
        "88.6",
        "76.9",
        "75.6",
        "80.1",
        "80.6",
        "88.7",
        "81.9",
        "82.4",
        "84.3",
        "84.9",
        "91.3"
      ],
      "column_index": 1,
      "raw_value": "88.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini",
    "bench": "math",
    "score": 0.64,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "source_id": "card-microsoft-phi-4-mini-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4 mini-Ins column; Instruct checkpoint. Shot count/CoT are in each row label. Appendix A allows output-format-specific system prompts but keeps few-shot examples fixed. GPQA subset unspecified. Overall composite excluded. Row: MATH (0-shot, CoT). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MATH (0-shot, CoT)",
      "column_label": "Similar size",
      "row": [
        "MATH (0-shot, CoT)",
        "64.0",
        "49.8",
        "46.7",
        "41.8",
        "61.7",
        "60.4",
        "41.6",
        "47.6",
        "46.1",
        "51.3",
        "70.2"
      ],
      "column_index": 1,
      "raw_value": "64.0",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "aime-2024",
    "score": 0.575,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "source_id": "card-microsoft-phi-4-mini-reasoning",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Phi-4-mini-reasoning checkpoint. Numerical source is the MIT-licensed model card. Its linked report https://arxiv.org/html/2504.21233v1#S2.T1 resolves the otherwise ambiguous AIME label to AIME 2024 for the identical model/evaluation row. The report is used only to disambiguate methodology; its non-exclusive licence is not treated as a data-reuse grant. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Phi-4-mini-reasoning (3.8B)",
      "column_label": "AIME",
      "row": [
        "Phi-4-mini-reasoning (3.8B)",
        "57.5",
        "94.6",
        "52.0"
      ],
      "column_index": 1,
      "raw_value": "57.5",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "math-500",
    "score": 0.946,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "source_id": "card-microsoft-phi-4-mini-reasoning",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Phi-4-mini-reasoning checkpoint. Numerical source is the MIT-licensed model card. Its linked report https://arxiv.org/html/2504.21233v1#S2.T1 resolves the otherwise ambiguous AIME label to AIME 2024 for the identical model/evaluation row. The report is used only to disambiguate methodology; its non-exclusive licence is not treated as a data-reuse grant. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Phi-4-mini-reasoning (3.8B)",
      "column_label": "MATH-500",
      "row": [
        "Phi-4-mini-reasoning (3.8B)",
        "57.5",
        "94.6",
        "52.0"
      ],
      "column_index": 2,
      "raw_value": "94.6",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.52,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "source_id": "card-microsoft-phi-4-mini-reasoning",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Phi-4-mini-reasoning checkpoint. Numerical source is the MIT-licensed model card. Its linked report https://arxiv.org/html/2504.21233v1#S2.T1 resolves the otherwise ambiguous AIME label to AIME 2024 for the identical model/evaluation row. The report is used only to disambiguate methodology; its non-exclusive licence is not treated as a data-reuse grant. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Phi-4-mini-reasoning (3.8B)",
      "column_label": "GPQA Diamond",
      "row": [
        "Phi-4-mini-reasoning (3.8B)",
        "57.5",
        "94.6",
        "52.0"
      ],
      "column_index": 3,
      "raw_value": "52.0",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "phi-4",
    "bench": "mmlu",
    "score": 0.848,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact phi-4 (14B) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "MMLU",
      "column_label": "phi-4 (14B)",
      "row": [
        "Popular Aggregated Benchmark",
        "MMLU",
        "84.8",
        "77.9",
        "79.9",
        "81.8",
        "86.3",
        "85.3",
        "88.1"
      ],
      "column_index": 2,
      "raw_value": "84.8",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4",
    "bench": "gpqa-unspecified",
    "score": 0.561,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact phi-4 (14B) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "GPQA",
      "column_label": "phi-4 (14B)",
      "row": [
        "Science",
        "GPQA",
        "56.1",
        "31.2",
        "42.9",
        "40.9",
        "49.1",
        "49.0",
        "50.6"
      ],
      "column_index": 2,
      "raw_value": "56.1",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4",
    "bench": "mgsm",
    "score": 0.806,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation; two line-broken benchmark labels MGSM then MATH align with two values in the same HTML cell. Exact phi-4 (14B) column. MATH subset unresolved in this card and not asserted to be the full test or MATH-500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table-multiline-cell",
      "table_index": 2,
      "row_label": "MGSM MATH",
      "column_label": "phi-4 (14B)",
      "row": [
        "Math",
        "MGSM MATH",
        "80.6 80.4",
        "53.5 44.6",
        "79.6 75.6",
        "86.5 73.0",
        "89.1 66.3*",
        "87.3 80.0",
        "90.4 74.6"
      ],
      "column_index": 2,
      "value_index": 0,
      "raw_value": "80.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4",
    "bench": "math-subset-unspecified",
    "score": 0.804,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation; two line-broken benchmark labels MGSM then MATH align with two values in the same HTML cell. Exact phi-4 (14B) column. MATH subset unresolved in this card and not asserted to be the full test or MATH-500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table-multiline-cell",
      "table_index": 2,
      "row_label": "MGSM MATH",
      "column_label": "phi-4 (14B)",
      "row": [
        "Math",
        "MGSM MATH",
        "80.6 80.4",
        "53.5 44.6",
        "79.6 75.6",
        "86.5 73.0",
        "89.1 66.3*",
        "87.3 80.0",
        "90.4 74.6"
      ],
      "column_index": 2,
      "value_index": 1,
      "raw_value": "80.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4",
    "bench": "humaneval",
    "score": 0.826,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact phi-4 (14B) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "HumanEval",
      "column_label": "phi-4 (14B)",
      "row": [
        "Code Generation",
        "HumanEval",
        "82.6",
        "67.8",
        "72.1",
        "86.2",
        "78.9*",
        "80.4",
        "90.6"
      ],
      "column_index": 2,
      "raw_value": "82.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "phi-4",
    "bench": "simpleqa",
    "score": 0.03,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact phi-4 (14B) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "SimpleQA",
      "column_label": "phi-4 (14B)",
      "row": [
        "Factual Knowledge",
        "SimpleQA",
        "3.0",
        "7.6",
        "5.4",
        "9.9",
        "20.9",
        "10.2",
        "39.4"
      ],
      "column_index": 2,
      "raw_value": "3.0",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "mmlu",
    "score": 0.799,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact Qwen 2.5 (14B instruct) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "MMLU",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Popular Aggregated Benchmark",
        "MMLU",
        "84.8",
        "77.9",
        "79.9",
        "81.8",
        "86.3",
        "85.3",
        "88.1"
      ],
      "column_index": 4,
      "raw_value": "79.9",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "gpqa-unspecified",
    "score": 0.429,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact Qwen 2.5 (14B instruct) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "GPQA",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Science",
        "GPQA",
        "56.1",
        "31.2",
        "42.9",
        "40.9",
        "49.1",
        "49.0",
        "50.6"
      ],
      "column_index": 4,
      "raw_value": "42.9",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "mgsm",
    "score": 0.796,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation; two line-broken benchmark labels MGSM then MATH align with two values in the same HTML cell. Exact Qwen 2.5 (14B instruct) column. MATH subset unresolved in this card and not asserted to be the full test or MATH-500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table-multiline-cell",
      "table_index": 2,
      "row_label": "MGSM MATH",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Math",
        "MGSM MATH",
        "80.6 80.4",
        "53.5 44.6",
        "79.6 75.6",
        "86.5 73.0",
        "89.1 66.3*",
        "87.3 80.0",
        "90.4 74.6"
      ],
      "column_index": 4,
      "value_index": 0,
      "raw_value": "79.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "math-subset-unspecified",
    "score": 0.756,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation; two line-broken benchmark labels MGSM then MATH align with two values in the same HTML cell. Exact Qwen 2.5 (14B instruct) column. MATH subset unresolved in this card and not asserted to be the full test or MATH-500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table-multiline-cell",
      "table_index": 2,
      "row_label": "MGSM MATH",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Math",
        "MGSM MATH",
        "80.6 80.4",
        "53.5 44.6",
        "79.6 75.6",
        "86.5 73.0",
        "89.1 66.3*",
        "87.3 80.0",
        "90.4 74.6"
      ],
      "column_index": 4,
      "value_index": 1,
      "raw_value": "75.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "humaneval",
    "score": 0.721,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact Qwen 2.5 (14B instruct) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "HumanEval",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Code Generation",
        "HumanEval",
        "82.6",
        "67.8",
        "72.1",
        "86.2",
        "78.9*",
        "80.4",
        "90.6"
      ],
      "column_index": 4,
      "raw_value": "72.1",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simpleqa",
    "score": 0.054,
    "source": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md",
    "source_id": "card-microsoft-phi-4",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Microsoft evaluation, exact Qwen 2.5 (14B instruct) column. Card does not state all shots/decoding settings. GPQA is not relabelled Diamond. DROP scoring rule is not stated in the card and is not asserted to be F1 or exact match. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "SimpleQA",
      "column_label": "Qwen 2.5 (14B instruct)",
      "row": [
        "Factual Knowledge",
        "SimpleQA",
        "3.0",
        "7.6",
        "5.4",
        "9.9",
        "20.9",
        "10.2",
        "39.4"
      ],
      "column_index": 4,
      "raw_value": "5.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "aime-2024",
    "score": 0.716,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "AIME 2024",
      "column_index": 3,
      "row": [
        "AIME 2024",
        "69.6",
        "74.6",
        "71.6",
        "67.7",
        "72.1",
        "54.9",
        "74.0",
        "70.9",
        "77.0"
      ],
      "raw_value": "71.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "aime-2025",
    "score": 0.646,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "AIME 2025",
      "column_index": 3,
      "row": [
        "AIME 2025",
        "57.6",
        "62.7",
        "64.6",
        "57.2",
        "58.9",
        "40.2",
        "67.8",
        "61.5",
        "73.1"
      ],
      "raw_value": "64.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "bbh",
    "score": 0.866,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "BigBenchHard",
      "column_index": 3,
      "row": [
        "BigBenchHard",
        "84.1",
        "83.7",
        "86.6",
        "77.1",
        "86.2",
        "73.5",
        "84.4",
        "86.8",
        "81.3"
      ],
      "raw_value": "86.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "zebralogic",
    "score": 0.665,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "ZebraLogic",
      "column_index": 3,
      "row": [
        "ZebraLogic",
        "57.9",
        "60.6",
        "66.5",
        "34.9",
        "60.8",
        "26.1",
        "85.2",
        "91.2",
        "22.4"
      ],
      "raw_value": "66.5",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "agieval-en",
    "score": 0.815,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "AGI Eval English",
      "column_index": 3,
      "row": [
        "AGI Eval English",
        "77.2",
        "79.1",
        "81.5",
        "78.6",
        "83.1",
        "69.5",
        "87.0",
        "90.1",
        "81.4"
      ],
      "raw_value": "81.5",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "humaneval-plus",
    "score": 0.899,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "HumanEvalPlus",
      "column_index": 3,
      "row": [
        "HumanEvalPlus",
        "88.2",
        "91.4",
        "89.9",
        "87.4",
        "89.7",
        "83.0",
        "80.2",
        "83.7",
        "89.7"
      ],
      "raw_value": "89.9",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "mbpp-plus",
    "score": 0.647,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "MBPP+",
      "column_index": 3,
      "row": [
        "MBPP+",
        "63.2",
        "63.0",
        "64.7",
        "61.4",
        "66.1",
        "63.5",
        "69.1",
        "63.0",
        "61.2"
      ],
      "raw_value": "64.7",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "livecodebench-v3",
    "score": 0.752,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "LiveCodeBench v3",
      "column_index": 3,
      "row": [
        "LiveCodeBench v3",
        "67.8",
        "75.1",
        "75.2",
        "68.0",
        "83.4",
        "58.8",
        "86.2",
        "85.5",
        "82.3"
      ],
      "raw_value": "75.2",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "ifeval-unspecified",
    "score": 0.882,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "IFEval",
      "column_index": 3,
      "row": [
        "IFEval",
        "77.9",
        "75.9",
        "88.2",
        "51.7",
        "86.0",
        "59.6",
        "87.4",
        "85.5",
        "42.5"
      ],
      "raw_value": "88.2",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "ifbench-unspecified",
    "score": 0.416,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "IFBench",
      "column_index": 3,
      "row": [
        "IFBench",
        "30.0",
        "28.3",
        "41.6",
        "23.0",
        "34.6",
        "16.7",
        "37.1",
        "40.4",
        "23.4"
      ],
      "raw_value": "41.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "mmlu",
    "score": 0.778,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "MMLU",
      "column_index": 3,
      "row": [
        "MMLU",
        "74.9",
        "74.8",
        "77.8",
        "77.4",
        "84.3",
        "67.9",
        "85.4",
        "86.5",
        "80.7"
      ],
      "raw_value": "77.8",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "popqa",
    "score": 0.237,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "PopQA",
      "column_index": 3,
      "row": [
        "PopQA",
        "20.8",
        "24.7",
        "23.7",
        "18.0",
        "17.9",
        "12.8",
        "24.3",
        "29.3",
        "14.5"
      ],
      "raw_value": "23.7",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "gpqa-unspecified",
    "score": 0.462,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "GPQA",
      "column_index": 3,
      "row": [
        "GPQA",
        "45.8",
        "48.6",
        "46.2",
        "47.6",
        "56.2",
        "54.4",
        "57.7",
        "61.5",
        "56.6"
      ],
      "raw_value": "46.2",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "alpacaeval2-lc",
    "score": 0.521,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 15 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T15",
      "row_label": "AlpacaEval 2 LC",
      "column_index": 3,
      "row": [
        "AlpacaEval 2 LC",
        "43.9",
        "50.6",
        "52.1",
        "24.0",
        "58.0",
        "7.7",
        "60.5",
        "73.5",
        "8.6"
      ],
      "raw_value": "52.1",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "aime-2024",
    "score": 0.443,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "AIME 2024",
      "column_index": 3,
      "row": [
        "AIME 2024",
        "6.7",
        "23.5",
        "44.3",
        "26.2",
        "55.1",
        "11.3",
        "1.3",
        "0.5",
        "7.3"
      ],
      "raw_value": "44.3",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "aime-2025",
    "score": 0.325,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "AIME 2025",
      "column_index": 3,
      "row": [
        "AIME 2025",
        "7.2",
        "20.4",
        "32.5",
        "21.7",
        "43.3",
        "6.3",
        "0.4",
        "0.2",
        "6.3"
      ],
      "raw_value": "32.5",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "bbh",
    "score": 0.712,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "BigBenchHard",
      "column_index": 3,
      "row": [
        "BigBenchHard",
        "51.0",
        "69.3",
        "71.2",
        "73.7",
        "85.6",
        "68.8",
        "43.8",
        "42.2",
        "61.2"
      ],
      "raw_value": "71.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "zebralogic",
    "score": 0.329,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "ZebraLogic",
      "column_index": 3,
      "row": [
        "ZebraLogic",
        "18.0",
        "28.4",
        "32.9",
        "25.4",
        "64.3",
        "10.7",
        "5.3",
        "5.3",
        "17.6"
      ],
      "raw_value": "32.9",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "agieval-en",
    "score": 0.644,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "AGI Eval English",
      "column_index": 3,
      "row": [
        "AGI Eval English",
        "59.2",
        "64.0",
        "64.4",
        "76.0",
        "84.5",
        "69.8",
        "56.1",
        "50.8",
        "64.0"
      ],
      "raw_value": "64.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "humaneval-plus",
    "score": 0.772,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "HumanEvalPlus",
      "column_index": 3,
      "row": [
        "HumanEvalPlus",
        "69.8",
        "72.9",
        "77.2",
        "79.8",
        "82.9",
        "74.9",
        "25.8",
        "34.4",
        "64.0"
      ],
      "raw_value": "77.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "mbpp-plus",
    "score": 0.602,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "MBPP+",
      "column_index": 3,
      "row": [
        "MBPP+",
        "56.5",
        "55.9",
        "60.2",
        "64.4",
        "66.3",
        "62.6",
        "40.7",
        "42.1",
        "54.0"
      ],
      "raw_value": "60.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "livecodebench-v3",
    "score": 0.295,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "LiveCodeBench v3",
      "column_index": 3,
      "row": [
        "LiveCodeBench v3",
        "20.0",
        "18.8",
        "29.5",
        "53.2",
        "55.9",
        "34.5",
        "7.2",
        "7.8",
        "11.5"
      ],
      "raw_value": "29.5",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.856,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "IFEval",
      "column_index": 3,
      "row": [
        "IFEval",
        "81.7",
        "82.0",
        "85.6",
        "86.3",
        "87.8",
        "73.4",
        "72.2",
        "71.4",
        "77.5"
      ],
      "raw_value": "85.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "ifbench-unspecified",
    "score": 0.323,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "IFBench",
      "column_index": 3,
      "row": [
        "IFBench",
        "27.4",
        "29.3",
        "32.3",
        "29.3",
        "34.0",
        "28.4",
        "26.7",
        "22.1",
        "22.3"
      ],
      "raw_value": "32.3",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "mmlu",
    "score": 0.691,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "MMLU",
      "column_index": 3,
      "row": [
        "MMLU",
        "67.1",
        "69.1",
        "69.1",
        "80.4",
        "83.6",
        "77.2",
        "61.6",
        "62.7",
        "63.5"
      ],
      "raw_value": "69.1",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "popqa",
    "score": 0.141,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "PopQA",
      "column_index": 3,
      "row": [
        "PopQA",
        "16.5",
        "20.7",
        "14.1",
        "20.4",
        "26.5",
        "21.5",
        "25.5",
        "25.5",
        "28.9"
      ],
      "raw_value": "14.1",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.404,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "GPQA",
      "column_index": 3,
      "row": [
        "GPQA",
        "30.0",
        "37.9",
        "40.4",
        "44.6",
        "51.1",
        "35.6",
        "31.3",
        "28.8",
        "33.0"
      ],
      "raw_value": "40.4",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.409,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 26 final checkpoint column (instruct), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S5.T26",
      "row_label": "AlpacaEval 2 LC",
      "column_index": 3,
      "row": [
        "AlpacaEval 2 LC",
        "21.8",
        "43.3",
        "40.9",
        "49.8",
        "73.5",
        "23.0",
        "18.3",
        "8.1",
        "28.6"
      ],
      "raw_value": "40.9",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "aime-2024",
    "score": 0.806,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "AIME 2024",
      "column_index": 4,
      "row": [
        "AIME 2024",
        "73.5",
        "76.0",
        "76.8",
        "80.6",
        "80.8",
        "86.3",
        "70.3",
        "78.4"
      ],
      "raw_value": "80.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "aime-2025",
    "score": 0.781,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "AIME 2025",
      "column_index": 4,
      "row": [
        "AIME 2025",
        "66.2",
        "70.7",
        "72.5",
        "78.1",
        "70.9",
        "78.8",
        "56.3",
        "70.3"
      ],
      "raw_value": "78.1",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "bbh",
    "score": 0.886,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "BigBenchHard",
      "column_index": 4,
      "row": [
        "BigBenchHard",
        "88.8",
        "89.1",
        "89.8",
        "88.6",
        "90.6",
        "91.1",
        "89.7",
        "87.6"
      ],
      "raw_value": "88.6",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "zebralogic",
    "score": 0.801,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "ZebraLogic",
      "column_index": 4,
      "row": [
        "ZebraLogic",
        "70.5",
        "74.5",
        "76.0",
        "80.1",
        "88.3",
        "96.1",
        "69.4",
        "79.2"
      ],
      "raw_value": "80.1",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "agieval-en",
    "score": 0.888,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "AGI Eval English",
      "column_index": 4,
      "row": [
        "AGI Eval English",
        "85.9",
        "87.8",
        "88.2",
        "88.8",
        "90.0",
        "92.2",
        "88.1",
        "89.6"
      ],
      "raw_value": "88.8",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "humaneval-plus",
    "score": 0.915,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "HumanEvalPlus",
      "column_index": 4,
      "row": [
        "HumanEvalPlus",
        "90.0",
        "91.6",
        "91.4",
        "91.5",
        "91.2",
        "90.6",
        "92.3",
        "88.0"
      ],
      "raw_value": "91.5",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "mbpp-plus",
    "score": 0.683,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "MBPP+",
      "column_index": 4,
      "row": [
        "MBPP+",
        "66.7",
        "67.2",
        "68.0",
        "68.3",
        "70.6",
        "66.2",
        "70.1",
        "66.0"
      ],
      "raw_value": "68.3",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "livecodebench-v3",
    "score": 0.833,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "LiveCodeBench v3",
      "column_index": 4,
      "row": [
        "LiveCodeBench v3",
        "75.8",
        "81.9",
        "83.5",
        "83.3",
        "90.2",
        "84.8",
        "79.5",
        "78.4"
      ],
      "raw_value": "83.3",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "ifeval-unspecified",
    "score": 0.938,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "IFEval",
      "column_index": 4,
      "row": [
        "IFEval",
        "83.9",
        "80.6",
        "89.0",
        "93.8",
        "86.5",
        "85.5",
        "78.7",
        "68.7"
      ],
      "raw_value": "93.8",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "ifbench-unspecified",
    "score": 0.681,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "IFBench",
      "column_index": 4,
      "row": [
        "IFBench",
        "37.0",
        "34.4",
        "47.6",
        "68.1",
        "37.3",
        "55.1",
        "23.8",
        "46.3"
      ],
      "raw_value": "68.1",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "mmlu",
    "score": 0.864,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "MMLU",
      "column_index": 4,
      "row": [
        "MMLU",
        "85.3",
        "85.2",
        "85.4",
        "86.4",
        "88.8",
        "90.1",
        "88.0",
        "88.4"
      ],
      "raw_value": "86.4",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "popqa",
    "score": 0.309,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "PopQA",
      "column_index": 4,
      "row": [
        "PopQA",
        "33.1",
        "37.0",
        "31.9",
        "30.9",
        "30.7",
        "32.2",
        "26.7",
        "32.2"
      ],
      "raw_value": "30.9",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "gpqa-unspecified",
    "score": 0.567,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "GPQA",
      "column_index": 4,
      "row": [
        "GPQA",
        "55.7",
        "57.6",
        "58.1",
        "56.7",
        "67.3",
        "67.4",
        "61.8",
        "64.0"
      ],
      "raw_value": "56.7",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "alpacaeval2-lc",
    "score": 0.691,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 final checkpoint column (think), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH omitted here because the report lists both full MATH and MATH-500 in methods without resolving this row. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "LaTeXML-span-table",
      "table_id": "S4.T14",
      "row_label": "AlpacaEval 2 LC",
      "column_index": 4,
      "row": [
        "AlpacaEval 2 LC",
        "69.1",
        "78.6",
        "74.2",
        "69.1",
        "75.6",
        "80.9",
        "26.2",
        "-"
      ],
      "raw_value": "69.1",
      "scale_divisor": 100
    },
    "mode": "think"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "mmlu",
    "score": 0.558,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "MMLU",
      "column_index": 2,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "55.8",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "bbh",
    "score": 0.415,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "BBH",
      "column_index": 3,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "41.5",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "gpqa-unspecified",
    "score": 0.272,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "GPQA",
      "column_index": 4,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "27.2",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "zebralogic",
    "score": 0.298,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "Zebra",
      "column_index": 5,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "29.8",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "aime-2025",
    "score": 0.152,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "AIME25",
      "column_index": 7,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "15.2",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "aime-2024",
    "score": 0.112,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "AIME24",
      "column_index": 8,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "11.2",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "humaneval-plus",
    "score": 0.148,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "CHE",
      "column_index": 9,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "14.8",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "livecodebench-v3",
    "score": 0.344,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "LCB",
      "column_index": 10,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "34.4",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "ifeval-unspecified",
    "score": 0.623,
    "source": "https://arxiv.org/html/2512.13961v2",
    "source_id": "report-2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ai2 evaluation of Qwen3 0.6B, the rejected-response generator in the DPO-teacher selection table. This is the named Qwen checkpoint, not an Olmo finetune. Table 16 zero-shot CoT protocol; 32768 max tokens, temperature 0.6, top_p 0.95. CHE is the report’s HumanEval+ shorthand; MATH omitted for subset ambiguity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T21",
      "row_label": "Qwen3 0.6B (rejected)",
      "column_label": "IFEval",
      "column_index": 11,
      "row": [
        "Qwen3 0.6B (rejected)",
        "35.1",
        "55.8",
        "41.5",
        "27.2",
        "29.8",
        "59.2",
        "15.2",
        "11.2",
        "14.8",
        "34.4",
        "62.3"
      ],
      "raw_value": "62.3",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "aime-2024",
    "score": 0.881,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "row_label": "Falcon-H1R-7B",
      "column_label": "AIME24",
      "row": [
        "Falcon-H1R-7B",
        "88.1",
        "83.1",
        "64.9",
        "36.3",
        "97.4"
      ],
      "column_index": 1,
      "raw_value": "88.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "aime-2025",
    "score": 0.831,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "row_label": "Falcon-H1R-7B",
      "column_label": "AIME25",
      "row": [
        "Falcon-H1R-7B",
        "88.1",
        "83.1",
        "64.9",
        "36.3",
        "97.4"
      ],
      "column_index": 2,
      "raw_value": "83.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.649,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "row_label": "Falcon-H1R-7B",
      "column_label": "HMMT25",
      "row": [
        "Falcon-H1R-7B",
        "88.1",
        "83.1",
        "64.9",
        "36.3",
        "97.4"
      ],
      "column_index": 3,
      "raw_value": "64.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "math-500",
    "score": 0.974,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "row_label": "Falcon-H1R-7B",
      "column_label": "MATH500",
      "row": [
        "Falcon-H1R-7B",
        "88.1",
        "83.1",
        "64.9",
        "36.3",
        "97.4"
      ],
      "column_index": 5,
      "raw_value": "97.4",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.686,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "row_label": "Falcon-H1R-7B",
      "column_label": "LCB v6",
      "row": [
        "Falcon-H1R-7B",
        "68.6",
        "28.3 / 3.9",
        "25.4",
        "4.9"
      ],
      "column_index": 1,
      "raw_value": "68.6",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "gpqa-diamond",
    "score": 0.613,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "row_label": "Falcon-H1R-7B",
      "column_label": "GPQA-Dimond",
      "row": [
        "Falcon-H1R-7B",
        "61.3",
        "72.1",
        "11.1",
        "53.4"
      ],
      "column_index": 1,
      "raw_value": "61.3",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "mmlu-pro",
    "score": 0.721,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "row_label": "Falcon-H1R-7B",
      "column_label": "MMLU-Pro",
      "row": [
        "Falcon-H1R-7B",
        "61.3",
        "72.1",
        "11.1",
        "53.4"
      ],
      "column_index": 2,
      "raw_value": "72.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "hle",
    "score": 0.111,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "row_label": "Falcon-H1R-7B",
      "column_label": "HLE",
      "row": [
        "Falcon-H1R-7B",
        "61.3",
        "72.1",
        "11.1",
        "53.4"
      ],
      "column_index": 3,
      "raw_value": "11.1",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "ifbench-unspecified",
    "score": 0.534,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Falcon-H1R-7B row, regular sampling (not DeepConf test-time scaling). Table 3: AIME/HMMT 16 responses and 65536 tokens; GPQA 5/32768; LCB v6 3/65536; MMLU-Pro 3/32768; HLE 1/65536. No AA-starred comparison cells imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "row_label": "Falcon-H1R-7B",
      "column_label": "IFBench",
      "row": [
        "Falcon-H1R-7B",
        "61.3",
        "72.1",
        "11.1",
        "53.4"
      ],
      "column_index": 4,
      "raw_value": "53.4",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "scicode-main",
    "score": 0.039,
    "source": "https://arxiv.org/html/2601.02346v1",
    "source_id": "report-2601.02346v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "SciCode main-problem score only (3.9); source also lists 28.3 subproblems, deliberately not imported. Table 3: one response, 65536 max tokens. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "row_label": "Falcon-H1R-7B",
      "column_label": "SciCode sub / main",
      "row": [
        "Falcon-H1R-7B",
        "68.6",
        "28.3 / 3.9",
        "25.4",
        "4.9"
      ],
      "column_index": 2,
      "raw_value": " 3.9",
      "scale_divisor": 100
    },
    "mode": "reasoning"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2025",
    "score": 0.721,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: AIME25. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "AIME25",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "AIME25",
        "69.3%",
        "72.1%"
      ],
      "column_index": 2,
      "raw_value": "72.1%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "math-500",
    "score": 0.978,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: MATH500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MATH500",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "MATH500",
        "96.3%",
        "97.8%"
      ],
      "column_index": 2,
      "raw_value": "97.8%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "gpqa-unspecified",
    "score": 0.64,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: GPQA. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "GPQA",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "GPQA",
        "59.6%",
        "64.0%"
      ],
      "column_index": 2,
      "raw_value": "64.0%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-unversioned",
    "score": 0.711,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: LCB. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "LCB",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "LCB",
        "59.5%",
        "71.1%"
      ],
      "column_index": 2,
      "raw_value": "71.1%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "bfcl-v3",
    "score": 0.669,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: BFCL v3. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "BFCL v3",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "BFCL v3",
        "66.3%",
        "66.9%"
      ],
      "column_index": 2,
      "raw_value": "66.9%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "ifeval-instruction-strict",
    "score": 0.903,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: IFEval (Instruction Strict). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "IFEval (Instruction Strict)",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "IFEval (Instruction Strict)",
        "89.4%",
        "90.3%"
      ],
      "column_index": 2,
      "raw_value": "90.3%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "hle",
    "score": 0.065,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-nano-9b-v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Reasoning-On evaluation using NeMo-Skills. The card abbreviates GPQA and LCB; no subset/window inferred. Full licence includes documentation and permits commercial reuse. Row: HLE. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HLE",
      "column_label": "NVIDIA-Nemotron-Nano-9B-v2",
      "row": [
        "HLE",
        "4.4%",
        "6.5%"
      ],
      "column_index": 2,
      "raw_value": "6.5%",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "aime-2025",
    "score": 0.785,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: AIME25. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "AIME25",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "AIME25",
        "78.5"
      ],
      "column_index": 1,
      "raw_value": "78.5",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "math-500",
    "score": 0.954,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: MATH500. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "MATH500",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "MATH500",
        "95.4"
      ],
      "column_index": 1,
      "raw_value": "95.4",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "gpqa-unspecified",
    "score": 0.532,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: GPQA. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "GPQA",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "GPQA",
        "53.2"
      ],
      "column_index": 1,
      "raw_value": "53.2",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "livecodebench-unversioned",
    "score": 0.518,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: LCB. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "LCB",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "LCB",
        "51.8"
      ],
      "column_index": 1,
      "raw_value": "51.8",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "bfcl-v3",
    "score": 0.611,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: BFCL v3. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "BFCL v3",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "BFCL v3",
        "61.1"
      ],
      "column_index": 1,
      "raw_value": "61.1",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "ifeval-prompt-unspecified-strictness",
    "score": 0.879,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-4b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "BF16 checkpoint, Reasoning-On section, NeMo-Skills. IFEVAL-Prompt 87.9 retained; overlapping instruction-level 92.0 not also added. Strict/loose and GPQA/LCB subset/window are not stated. Row: IFEVAL-Prompt. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "IFEVAL-Prompt",
      "column_label": "NVIDIA-Nemotron-3-Nano-4B-BF16",
      "row": [
        "IFEVAL-Prompt",
        "87.9"
      ],
      "column_index": 1,
      "raw_value": "87.9",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "aime-2025",
    "score": 0.924,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: AIME 2025. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "AIME 2025",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "AIME 2025",
        "89.1",
        "90.2",
        "91.9‡",
        "92.4 (98.6)†"
      ],
      "column_index": 4,
      "raw_value": "92.4 (98.6)†",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "aime-2026",
    "score": 0.909,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: AIME 2026. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "AIME 2026",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "AIME 2026",
        "89.9‡",
        "89.8‡",
        "91.1‡",
        "90.9 (95.0)†"
      ],
      "column_index": 4,
      "raw_value": "90.9 (95.0)†",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "hmmt-feb-2025",
    "score": 0.946,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: HMMT Feb25. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HMMT Feb25",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "HMMT Feb25",
        "84.6‡",
        "93.7",
        "89.0",
        "94.6"
      ],
      "column_index": 4,
      "raw_value": "94.6",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.872,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: LiveCodeBench v6 (2408-2505). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "LiveCodeBench v6 (2408-2505)",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "LiveCodeBench v6 (2408-2505)",
        "68.3",
        "78.7",
        "74.6",
        "87.2 (88.4)†"
      ],
      "column_index": 4,
      "raw_value": "87.2 (88.4)†",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "mmlu-redux",
    "score": 0.863,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: MMLU-Redux. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MMLU-Redux",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "MMLU-Redux",
        "-",
        "-",
        "93.3",
        "86.3"
      ],
      "column_index": 4,
      "raw_value": "86.3",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.798,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: MMLU-Pro. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MMLU-Pro",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "MMLU-Pro",
        "78.3",
        "83.7",
        "85.3",
        "79.8"
      ],
      "column_index": 4,
      "raw_value": "79.8",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.761,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: GPQA-Diamond. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "GPQA-Diamond",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "GPQA-Diamond",
        "73.0",
        "79.2",
        "84.2",
        "76.1"
      ],
      "column_index": 4,
      "raw_value": "76.1",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "hle",
    "score": 0.177,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: HLE (no tool). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HLE (no tool)",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "HLE (no tool)",
        "10.6",
        "18.3",
        "22.4",
        "17.7"
      ],
      "column_index": 4,
      "raw_value": "17.7",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.835,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: ArenaHard v2 (Avg.). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "ArenaHard v2 (Avg.)",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "ArenaHard v2 (Avg.)",
        "67.7",
        "-",
        "65.4‡",
        "83.5"
      ],
      "column_index": 4,
      "raw_value": "83.5",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "ifbench-prompt",
    "score": 0.829,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: IFBench (prompt). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "IFBench (prompt)",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "IFBench (prompt)",
        "71.5",
        "72.6",
        "70.2",
        "82.9"
      ],
      "column_index": 4,
      "raw_value": "82.9",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "longbench-v2",
    "score": 0.403,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: LongBench v2. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "LongBench v2",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "LongBench v2",
        "39.6",
        "-",
        "59.0",
        "40.3"
      ],
      "column_index": 4,
      "raw_value": "40.3",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "bfcl-v4",
    "score": 0.529,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: BFCL v4. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "BFCL v4",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "BFCL v4",
        "53.8",
        "-",
        "67.3",
        "52.9"
      ],
      "column_index": 4,
      "raw_value": "52.9",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "tau2-bench",
    "score": 0.589,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: 𝜏²-Bench. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "𝜏²-Bench",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "𝜏²-Bench",
        "49.0",
        "61.2",
        "81.2",
        "58.9"
      ],
      "column_index": 4,
      "raw_value": "58.9",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "terminal-bench-2",
    "score": 0.211,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: Terminal Bench 2.0. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Terminal Bench 2.0",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "Terminal Bench 2.0",
        "8.5",
        "31.0",
        "40.5",
        "21.1"
      ],
      "column_index": 4,
      "raw_value": "21.1",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.502,
    "source": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md",
    "source_id": "card-nvidia-nemotron-cascade-2-30b-a3b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Cascade-2 column. Leading unparenthesized score selected; parenthesized † values use Tool-Integrated Reasoning and are not imported. ArenaHard aggregate only; AA-LCR omitted; no baseline columns. SWE scaffold OpenHands. Row: SWE Verified (OpenHands). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Verified (OpenHands)",
      "column_label": "Nemotron-Cascade-2-30B-A3B",
      "row": [
        "SWE Verified (OpenHands)",
        "38.8",
        "60.5",
        "69.2",
        "50.2"
      ],
      "column_index": 4,
      "raw_value": "50.2",
      "scale_divisor": 100
    },
    "mode": "thinking"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.783,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: MMLU-Pro. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "MMLU-Pro",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "MMLU-Pro",
        "78.3",
        "80.9",
        "75.0"
      ],
      "column_index": 1,
      "raw_value": "78.3",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "aime-2025",
    "score": 0.891,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: AIME25 (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "AIME25 (no tools)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "AIME25 (no tools)",
        "89.1",
        "85.0",
        "91.7"
      ],
      "column_index": 1,
      "raw_value": "89.1",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "gpqa-unspecified",
    "score": 0.73,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: GPQA (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "GPQA (no tools)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "GPQA (no tools)",
        "73.0",
        "73.4",
        "71.5"
      ],
      "column_index": 1,
      "raw_value": "73.0",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "scicode-subtask",
    "score": 0.333,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: SciCode (subtask). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SciCode (subtask)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "SciCode (subtask)",
        "33.3",
        "33.0",
        "34.0"
      ],
      "column_index": 1,
      "raw_value": "33.3",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "hle",
    "score": 0.106,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: HLE (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HLE (no tools)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "HLE (no tools)",
        "10.6",
        "9.8",
        "10.9"
      ],
      "column_index": 1,
      "raw_value": "10.6",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "tau2-bench",
    "score": 0.49,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: TauBench V2 (Average). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "TauBench V2 (Average)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "TauBench V2 (Average)",
        "49.0",
        "47.7",
        "48.7"
      ],
      "column_index": 1,
      "raw_value": "49.0",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "bfcl-v4",
    "score": 0.538,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: BFCL v4. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "BFCL v4",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "BFCL v4",
        "53.8",
        "46.4\\*",
        "\\-"
      ],
      "column_index": 1,
      "raw_value": "53.8",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "ifbench-prompt",
    "score": 0.715,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: IFBench (prompt). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "IFBench (prompt)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "IFBench (prompt)",
        "71.5",
        "51.0",
        "65.0"
      ],
      "column_index": 1,
      "raw_value": "71.5",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.677,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own BF16 column, reasoning evaluation. No-tool scores selected where paired. Recommended decoding temperature 1/top_p 1 for reasoning and 0.6/0.95 for tool calling; card recommendations are not claimed as per-row evaluation logs. LCB row withheld because date range is inverted (2025-08–2025-05). Tau and ArenaHard aggregates only. Row: Arena-Hard-V2 (Average). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Arena-Hard-V2 (Average)",
      "column_label": "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "row": [
        "Arena-Hard-V2 (Average)",
        "67.7",
        "57.8",
        "48.6"
      ],
      "column_index": 1,
      "raw_value": "67.7",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "mmlu-pro",
    "score": 0.8373,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: MMLU-Pro. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "MMLU-Pro",
      "column_label": "Nemotron 3 Super",
      "row": [
        "MMLU-Pro",
        "83.73",
        "86.70",
        "81.00"
      ],
      "column_index": 1,
      "raw_value": "83.73",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "aime-2025",
    "score": 0.9021,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: AIME25 (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "AIME25 (no tools)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "AIME25 (no tools)",
        "90.21",
        "90.36",
        "92.50"
      ],
      "column_index": 1,
      "raw_value": "90.21",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "hmmt-feb-2025",
    "score": 0.9367,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: HMMT Feb25 (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "HMMT Feb25 (no tools)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "HMMT Feb25 (no tools)",
        "93.67",
        "91.40",
        "90.00"
      ],
      "column_index": 1,
      "raw_value": "93.67",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "gpqa-unspecified",
    "score": 0.7923,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: GPQA (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "GPQA (no tools)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "GPQA (no tools)",
        "79.23",
        "86.60",
        "80.10"
      ],
      "column_index": 1,
      "raw_value": "79.23",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "livecodebench-v5-202407-202412",
    "score": 0.8119,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: LiveCodeBench (v5 2024-07↔2024-12). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "LiveCodeBench (v5 2024-07↔2024-12)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "LiveCodeBench (v5 2024-07↔2024-12)",
        "81.19",
        "78.93",
        "88.00"
      ],
      "column_index": 1,
      "raw_value": "81.19",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "scicode-subtask",
    "score": 0.4205,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: SciCode (subtask). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "SciCode (subtask)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "SciCode (subtask)",
        "42.05",
        "42.00",
        "39.00"
      ],
      "column_index": 1,
      "raw_value": "42.05",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "hle",
    "score": 0.1826,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: HLE (no tools). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "HLE (no tools)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "HLE (no tools)",
        "18.26",
        "25.30",
        "14.90"
      ],
      "column_index": 1,
      "raw_value": "18.26",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "terminal-bench-2",
    "score": 0.31,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: Terminal Bench Core 2.0. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Terminal Bench Core 2.0",
      "column_label": "Nemotron 3 Super",
      "row": [
        "Terminal Bench Core 2.0",
        "31.00",
        "37.50",
        "18.70"
      ],
      "column_index": 1,
      "raw_value": "31.00",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "swe-bench-multilingual",
    "score": 0.4578,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: SWE-Bench Multilingual (OpenHands). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "SWE-Bench Multilingual (OpenHands)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "SWE-Bench Multilingual (OpenHands)",
        "45.78",
        "—",
        "30.80"
      ],
      "column_index": 1,
      "raw_value": "45.78",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "tau2-bench",
    "score": 0.6115,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: Average. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Average",
      "column_label": "Nemotron 3 Super",
      "row": [
        "Average",
        "61.15",
        "74.53",
        "61.0"
      ],
      "column_index": 1,
      "raw_value": "61.15",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "ifbench-prompt",
    "score": 0.7256,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md",
    "source_id": "card-nvidia-nvidia-nemotron-3-super-120b-a12b-bf16",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Super BF16 column. No-tool scores selected where paired. Card recommends temperature 1.0/top_p .95 across tasks; not asserted as exact run logs. Tau V2 average selected without domain subrows; Terminal Bench Core 2.0, not hard-subset score. Row: IFBench (prompt). Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "IFBench (prompt)",
      "column_label": "Nemotron 3 Super",
      "row": [
        "IFBench (prompt)",
        "72.56",
        "73.77",
        "68.32"
      ],
      "column_index": 1,
      "raw_value": "72.56",
      "scale_divisor": 100
    },
    "mode": "reasoning-on"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.516,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "SWE-bench Verified w/ OpenHands, 100 turns",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "51.6",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-live",
    "score": 0.207,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "SWE-bench Live",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "20.7",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.347,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "SWE-bench Multilingual",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "34.7",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "aider-polyglot",
    "score": 0.333,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "Aider-Polyglot",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "33.3",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.622,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "BFCL-v3",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "62.2",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "tau-bench-retail",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "TAU-Bench Retail",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "68.7",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "tau-bench-airline",
    "score": 0.48,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Green Qwen3-Coder-30B-A3B-Instruct column in embedded chart. SWE Verified uses OpenHands 100 turns; identical 500-turn score not added again. Other harness details not specified. External image URL is content-hashed and tied to the pinned licensed README, not represented as an immutable Git image URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-30a3-main.jpg",
      "asset_sha256": "fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf",
      "label": "TAU-Bench Airline",
      "model_label": "Qwen3-Coder-30B-A3B-Instruct",
      "raw_value": "48.0",
      "scale_divisor": 100
    },
    "mode": "instruct"
  },
  {
    "model": "xtts-v2",
    "bench": "ttsds-v1-default",
    "score": 0.8733762543,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key xttsv2.tar.gz; submission has no inference settings.",
    "raw_value": 87.33762543065002,
    "scale_divisor": 100,
    "components": {
      "1": 94.30116158900522,
      "2": 87.80196183368757,
      "4": 72.5637905621263,
      "6": 94.68358773778098
    },
    "raw_rows": [
      {
        "benchmark_name": "Hubert",
        "benchmark_category": "1",
        "dataset": "xttsv2.tar.gz",
        "score": "94.08640627299717",
        "ci": "1.0",
        "time_taken": "134.5800392627716",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Wav2Vec2",
        "benchmark_category": "1",
        "dataset": "xttsv2.tar.gz",
        "score": "93.23599890583226",
        "ci": "1.0",
        "time_taken": "95.01738715171814",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WavLM",
        "benchmark_category": "1",
        "dataset": "xttsv2.tar.gz",
        "score": "95.58107958818624",
        "ci": "1.0",
        "time_taken": "95.93639731407166",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Hubert Token",
        "benchmark_category": "2",
        "dataset": "xttsv2.tar.gz",
        "score": "81.36422514825712",
        "ci": "1.0",
        "time_taken": "45.49807000160217",
        "noise_dataset": "noise_noise_uniform_distribution",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "MPM",
        "benchmark_category": "2",
        "dataset": "xttsv2.tar.gz",
        "score": "91.27897077788415",
        "ci": "1.0",
        "time_taken": "45.65868854522705",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Pitch",
        "benchmark_category": "2",
        "dataset": "xttsv2.tar.gz",
        "score": "90.76268957492144",
        "ci": "1.0",
        "time_taken": "19.80112147331237",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "VoiceFixer",
        "benchmark_category": "3",
        "dataset": "xttsv2.tar.gz",
        "score": "89.08521695705429",
        "ci": "1.0",
        "time_taken": "170.33265447616577",
        "noise_dataset": "noise_noise_all_ones",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "Wada SNR",
        "benchmark_category": "3",
        "dataset": "xttsv2.tar.gz",
        "score": "69.49957801623174",
        "ci": "1.0",
        "time_taken": "14.548080682754517",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "DVector",
        "benchmark_category": "4",
        "dataset": "xttsv2.tar.gz",
        "score": "71.560259547529",
        "ci": "1.0",
        "time_taken": "26.41282844543457",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WeSpeaker",
        "benchmark_category": "4",
        "dataset": "xttsv2.tar.gz",
        "score": "73.5673215767236",
        "ci": "1.0",
        "time_taken": "30.17014646530152",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Wav2Vec2 WER",
        "benchmark_category": "6",
        "dataset": "xttsv2.tar.gz",
        "score": "94.21111148710617",
        "ci": "1.0",
        "time_taken": "31.38028883934021",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "Whisper WER",
        "benchmark_category": "6",
        "dataset": "xttsv2.tar.gz",
        "score": "95.1560639884558",
        "ci": "1.0",
        "time_taken": "100.43997597694396",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_lj_speech"
      }
    ],
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/xttsv2.json"
  },
  {
    "model": "openvoice-v2",
    "bench": "ttsds-v1-default",
    "score": 0.8619282375,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key openvoice_v2.tar.gz; submission has no inference settings.",
    "raw_value": 86.19282374614762,
    "scale_divisor": 100,
    "components": {
      "1": 90.74575228952538,
      "2": 88.96598558279271,
      "4": 68.66945188613134,
      "6": 96.39010522614107
    },
    "raw_rows": [
      {
        "benchmark_name": "Hubert",
        "benchmark_category": "1",
        "dataset": "openvoice_v2.tar.gz",
        "score": "89.997296870775",
        "ci": "1.0",
        "time_taken": "119.05289673805235",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Wav2Vec2",
        "benchmark_category": "1",
        "dataset": "openvoice_v2.tar.gz",
        "score": "89.67547415210426",
        "ci": "1.0",
        "time_taken": "107.44602632522584",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WavLM",
        "benchmark_category": "1",
        "dataset": "openvoice_v2.tar.gz",
        "score": "92.5644858456969",
        "ci": "1.0",
        "time_taken": "88.64464712142944",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Hubert Token",
        "benchmark_category": "2",
        "dataset": "openvoice_v2.tar.gz",
        "score": "83.69874370210594",
        "ci": "1.0",
        "time_taken": "37.23568344116211",
        "noise_dataset": "noise_noise_uniform_distribution",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "MPM",
        "benchmark_category": "2",
        "dataset": "openvoice_v2.tar.gz",
        "score": "93.15773558583058",
        "ci": "1.0",
        "time_taken": "39.98830366134644",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Pitch",
        "benchmark_category": "2",
        "dataset": "openvoice_v2.tar.gz",
        "score": "90.04147746044163",
        "ci": "1.0",
        "time_taken": "16.07070231437683",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "VoiceFixer",
        "benchmark_category": "3",
        "dataset": "openvoice_v2.tar.gz",
        "score": "92.3373700440012",
        "ci": "1.0",
        "time_taken": "173.4297227859497",
        "noise_dataset": "noise_noise_all_ones",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "Wada SNR",
        "benchmark_category": "3",
        "dataset": "openvoice_v2.tar.gz",
        "score": "90.1427416170811",
        "ci": "1.0",
        "time_taken": "10.900976181030272",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "DVector",
        "benchmark_category": "4",
        "dataset": "openvoice_v2.tar.gz",
        "score": "68.35390860117379",
        "ci": "1.0",
        "time_taken": "21.824361085891724",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WeSpeaker",
        "benchmark_category": "4",
        "dataset": "openvoice_v2.tar.gz",
        "score": "68.9849951710889",
        "ci": "1.0",
        "time_taken": "30.21518111228943",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Wav2Vec2 WER",
        "benchmark_category": "6",
        "dataset": "openvoice_v2.tar.gz",
        "score": "98.12402858119476",
        "ci": "1.0",
        "time_taken": "25.35945439338684",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "Whisper WER",
        "benchmark_category": "6",
        "dataset": "openvoice_v2.tar.gz",
        "score": "94.65618187108738",
        "ci": "1.0",
        "time_taken": "94.4947431087494",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_lj_speech"
      }
    ],
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/openvoice_v2.json"
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "ttsds-v1-default",
    "score": 0.8072758827,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key metavoice-1b.tar.gz; Overwrote 30 second prompt constraint..",
    "raw_value": 80.72758826605822,
    "scale_divisor": 100,
    "components": {
      "1": 91.81513153476088,
      "2": 83.06492404027495,
      "4": 73.54640596870209,
      "6": 74.48389152049495
    },
    "raw_rows": [
      {
        "benchmark_name": "Hubert",
        "benchmark_category": "1",
        "dataset": "metavoice-1b.tar.gz",
        "score": "91.4462644751491",
        "ci": "1.0",
        "time_taken": "198.91358184814453",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Wav2Vec2",
        "benchmark_category": "1",
        "dataset": "metavoice-1b.tar.gz",
        "score": "90.89403858006676",
        "ci": "1.0",
        "time_taken": "174.4946646690369",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WavLM",
        "benchmark_category": "1",
        "dataset": "metavoice-1b.tar.gz",
        "score": "93.10509154906676",
        "ci": "1.0",
        "time_taken": "174.61395931243896",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Hubert Token",
        "benchmark_category": "2",
        "dataset": "metavoice-1b.tar.gz",
        "score": "70.86423834850312",
        "ci": "1.0",
        "time_taken": "114.71653652191162",
        "noise_dataset": "noise_normal_distribution",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "MPM",
        "benchmark_category": "2",
        "dataset": "metavoice-1b.tar.gz",
        "score": "87.63695939163797",
        "ci": "1.0",
        "time_taken": "54.0086669921875",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Pitch",
        "benchmark_category": "2",
        "dataset": "metavoice-1b.tar.gz",
        "score": "90.69357438068376",
        "ci": "1.0",
        "time_taken": "17.034030437469482",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "VoiceFixer",
        "benchmark_category": "3",
        "dataset": "metavoice-1b.tar.gz",
        "score": "85.32203699156116",
        "ci": "1.0",
        "time_taken": "620.2031667232513",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "Wada SNR",
        "benchmark_category": "3",
        "dataset": "metavoice-1b.tar.gz",
        "score": "79.9670145051596",
        "ci": "1.0",
        "time_taken": "13.67752742767334",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "DVector",
        "benchmark_category": "4",
        "dataset": "metavoice-1b.tar.gz",
        "score": "73.18305147841323",
        "ci": "1.0",
        "time_taken": "31.003464698791504",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WeSpeaker",
        "benchmark_category": "4",
        "dataset": "metavoice-1b.tar.gz",
        "score": "73.90976045899095",
        "ci": "1.0",
        "time_taken": "85.03007006645203",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Wav2Vec2 WER",
        "benchmark_category": "6",
        "dataset": "metavoice-1b.tar.gz",
        "score": "78.61354383179987",
        "ci": "1.0",
        "time_taken": "116.22621846199036",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_blizzard2008"
      },
      {
        "benchmark_name": "Whisper WER",
        "benchmark_category": "6",
        "dataset": "metavoice-1b.tar.gz",
        "score": "70.35423920919001",
        "ci": "1.0",
        "time_taken": "101.09989356994627",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      }
    ],
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/metavoice-1b.json"
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "ttsds-v1-default",
    "score": 0.8786419872,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key tortoise.tar.gz; used kv_cache.",
    "raw_value": 87.86419871586412,
    "scale_divisor": 100,
    "components": {
      "1": 95.04683346611588,
      "2": 86.53433803378029,
      "4": 74.14390362993673,
      "6": 95.73171973362358
    },
    "raw_rows": [
      {
        "benchmark_name": "Hubert",
        "benchmark_category": "1",
        "dataset": "tortoise.tar.gz",
        "score": "95.0032087870608",
        "ci": "1.0",
        "time_taken": "302.0166263580322",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Wav2Vec2",
        "benchmark_category": "1",
        "dataset": "tortoise.tar.gz",
        "score": "93.9126482849964",
        "ci": "1.0",
        "time_taken": "248.6831426620484",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WavLM",
        "benchmark_category": "1",
        "dataset": "tortoise.tar.gz",
        "score": "96.22464332629046",
        "ci": "1.0",
        "time_taken": "254.821049451828",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Hubert Token",
        "benchmark_category": "2",
        "dataset": "tortoise.tar.gz",
        "score": "74.8998668339101",
        "ci": "1.0",
        "time_taken": "172.94941592216492",
        "noise_dataset": "noise_normal_distribution",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "MPM",
        "benchmark_category": "2",
        "dataset": "tortoise.tar.gz",
        "score": "92.885228124336",
        "ci": "1.0",
        "time_taken": "73.89647388458252",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Pitch",
        "benchmark_category": "2",
        "dataset": "tortoise.tar.gz",
        "score": "91.81791914309478",
        "ci": "1.0",
        "time_taken": "28.67462491989136",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "VoiceFixer",
        "benchmark_category": "3",
        "dataset": "tortoise.tar.gz",
        "score": "89.3244590262843",
        "ci": "1.0",
        "time_taken": "677.6368510723114",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "Wada SNR",
        "benchmark_category": "3",
        "dataset": "tortoise.tar.gz",
        "score": "79.33163868919063",
        "ci": "1.0",
        "time_taken": "24.47475934028625",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_lj_speech"
      },
      {
        "benchmark_name": "DVector",
        "benchmark_category": "4",
        "dataset": "tortoise.tar.gz",
        "score": "71.62210142506505",
        "ci": "1.0",
        "time_taken": "44.36152744293213",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_test"
      },
      {
        "benchmark_name": "WeSpeaker",
        "benchmark_category": "4",
        "dataset": "tortoise.tar.gz",
        "score": "76.6657058348084",
        "ci": "1.0",
        "time_taken": "121.7685649394989",
        "noise_dataset": "noise_esc50",
        "reference_dataset": "reference_speech_libritts_r_test"
      },
      {
        "benchmark_name": "Wav2Vec2 WER",
        "benchmark_category": "6",
        "dataset": "tortoise.tar.gz",
        "score": "95.58743538661903",
        "ci": "1.0",
        "time_taken": "172.25165176391602",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_blizzard2013"
      },
      {
        "benchmark_name": "Whisper WER",
        "benchmark_category": "6",
        "dataset": "tortoise.tar.gz",
        "score": "95.87600408062814",
        "ci": "1.0",
        "time_taken": "104.27716970443726",
        "noise_dataset": "noise_all_ones",
        "reference_dataset": "reference_speech_blizzard2008"
      }
    ],
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/tortoise.json"
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1039,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Gemini-2.5-Flash-Preview-TTS; voice Zephyr; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 10.39,
    "scale_divisor": 100,
    "voice": "Zephyr",
    "strong_prompting": true
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.7557,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Gemini-2.5-Flash-Preview-TTS; voice Zephyr; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 75.57,
    "scale_divisor": 100,
    "voice": "Zephyr",
    "strong_prompting": true
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1179,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Gemini-2.5-Pro-Preview-TTS; voice Zephyr; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 11.79,
    "scale_divisor": 100,
    "voice": "Zephyr",
    "strong_prompting": true
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.7215,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Gemini-2.5-Pro-Preview-TTS; voice Zephyr; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 72.15,
    "scale_divisor": 100,
    "voice": "Zephyr",
    "strong_prompting": true
  },
  {
    "model": "higgs-audio-v2",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1945,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "lab",
    "date": "2025-08-10",
    "note": "README row Higgs Audio V2; voice Belinda; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 19.45,
    "scale_divisor": 100,
    "voice": "Belinda",
    "strong_prompting": false
  },
  {
    "model": "higgs-audio-v2",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.389,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "lab",
    "date": "2025-08-10",
    "note": "README row Higgs Audio V2; voice Belinda; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 38.9,
    "scale_divisor": 100,
    "voice": "Belinda",
    "strong_prompting": false
  },
  {
    "model": "minimax-speech-02-hd",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1007,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row minimax/speech-02-hd; voice English_expressive_narrator; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 10.07,
    "scale_divisor": 100,
    "voice": "English_expressive_narrator",
    "strong_prompting": false
  },
  {
    "model": "minimax-speech-02-hd",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.3677,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row minimax/speech-02-hd; voice English_expressive_narrator; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 36.77,
    "scale_divisor": 100,
    "voice": "English_expressive_narrator",
    "strong_prompting": false
  },
  {
    "model": "elevenlabs-multilingual-v2",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1119,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row 11Labs eleven multilingual v2; voice Brian; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 11.19,
    "scale_divisor": 100,
    "voice": "Brian",
    "strong_prompting": false
  },
  {
    "model": "elevenlabs-multilingual-v2",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.3364,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row 11Labs eleven multilingual v2; voice Brian; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 33.64,
    "scale_divisor": 100,
    "voice": "Brian",
    "strong_prompting": false
  },
  {
    "model": "kokoro-82m",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1341,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Kokoro-82M; voice af_heart; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 13.41,
    "scale_divisor": 100,
    "voice": "af_heart",
    "strong_prompting": false
  },
  {
    "model": "kokoro-82m",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.2589,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Kokoro-82M; voice af_heart; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 25.89,
    "scale_divisor": 100,
    "voice": "af_heart",
    "strong_prompting": false
  },
  {
    "model": "chatterbox",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1305,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row ResembleAI Chatterbox; voice -; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 13.05,
    "scale_divisor": 100,
    "voice": "-",
    "strong_prompting": false
  },
  {
    "model": "chatterbox",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.2449,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row ResembleAI Chatterbox; voice -; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 24.49,
    "scale_divisor": 100,
    "voice": "-",
    "strong_prompting": false
  },
  {
    "model": "zonos-v0-1",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1912,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Zyphra/Zonos; voice exampleaudio; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 19.12,
    "scale_divisor": 100,
    "voice": "exampleaudio",
    "strong_prompting": false
  },
  {
    "model": "zonos-v0-1",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.1407,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Zyphra/Zonos; voice exampleaudio; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 14.07,
    "scale_divisor": 100,
    "voice": "exampleaudio",
    "strong_prompting": false
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1076,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row gpt-4o-mini-tts; voice Alloy; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 10.76,
    "scale_divisor": 100,
    "voice": "Alloy",
    "strong_prompting": true
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.6024,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row gpt-4o-mini-tts; voice Alloy; strongly prompted. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 60.24,
    "scale_divisor": 100,
    "voice": "Alloy",
    "strong_prompting": true
  },
  {
    "model": "deepgram-aura-2",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1683,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row DeepGram Aura-2; voice Thalia-en; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 16.83,
    "scale_divisor": 100,
    "voice": "Thalia-en",
    "strong_prompting": false
  },
  {
    "model": "deepgram-aura-2",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.258,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row DeepGram Aura-2; voice Thalia-en; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 25.8,
    "scale_divisor": 100,
    "voice": "Thalia-en",
    "strong_prompting": false
  },
  {
    "model": "kyutai-tts-1-6b-en-fr",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1272,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row KyutAI-TTS; voice ex03-ex01_happy_001_channel1_334s; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 12.72,
    "scale_divisor": 100,
    "voice": "ex03-ex01_happy_001_channel1_334s",
    "strong_prompting": false
  },
  {
    "model": "kyutai-tts-1-6b-en-fr",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.2194,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row KyutAI-TTS; voice ex03-ex01_happy_001_channel1_334s; no strong-prompting star. Stable Gemini judge six-category overall column. API/weight build hashes not supplied.",
    "raw_value": 21.94,
    "scale_divisor": 100,
    "voice": "ex03-ex01_happy_001_channel1_334s",
    "strong_prompting": false
  },
  {
    "model": "voxtral-tts",
    "bench": "voxtral-tts-minimax9-wer",
    "score": 0.0191,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 1.9100000000000001,
    "scale_divisor": 100,
    "components": {
      "Arabic": 2.68,
      "German": 0.83,
      "English": 0.63,
      "Spanish": 0.51,
      "French": 3.22,
      "Hindi": 4.99,
      "Italian": 1.32,
      "Dutch": 1.99,
      "Portuguese": 1.02
    }
  },
  {
    "model": "voxtral-tts",
    "bench": "voxtral-tts-minimax9-speaker",
    "score": 0.7427777778,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 0.7427777777777778,
    "scale_divisor": 1,
    "components": {
      "Arabic": 0.746,
      "German": 0.721,
      "English": 0.786,
      "Spanish": 0.762,
      "French": 0.587,
      "Hindi": 0.839,
      "Italian": 0.739,
      "Dutch": 0.72,
      "Portuguese": 0.785
    }
  },
  {
    "model": "voxtral-tts",
    "bench": "voxtral-tts-seed-reported-wer",
    "score": 0.0123,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 1.23,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-tts",
    "bench": "voxtral-tts-seed-reported-speaker",
    "score": 0.628,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 0.628,
    "scale_divisor": 1
  },
  {
    "model": "elevenlabs-v3",
    "bench": "voxtral-tts-minimax9-wer",
    "score": 0.0217111111,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 2.171111111111111,
    "scale_divisor": 100,
    "components": {
      "Arabic": 3.67,
      "German": 0.45,
      "English": 0.48,
      "Spanish": 0.87,
      "French": 2.34,
      "Hindi": 8.71,
      "Italian": 0.58,
      "Dutch": 1.52,
      "Portuguese": 0.92
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "voxtral-tts-minimax9-speaker",
    "score": 0.4967777778,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 0.49677777777777776,
    "scale_divisor": 1,
    "components": {
      "Arabic": 0.546,
      "German": 0.457,
      "English": 0.484,
      "Spanish": 0.443,
      "French": 0.339,
      "Hindi": 0.707,
      "Italian": 0.527,
      "Dutch": 0.397,
      "Portuguese": 0.571
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "voxtral-tts-seed-reported-wer",
    "score": 0.0126,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 1.26,
    "scale_divisor": 100
  },
  {
    "model": "elevenlabs-v3",
    "bench": "voxtral-tts-seed-reported-speaker",
    "score": 0.392,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 0.392,
    "scale_divisor": 1
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "voxtral-tts-minimax9-wer",
    "score": 0.0162555556,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 1.6255555555555554,
    "scale_divisor": 100,
    "components": {
      "Arabic": 2.86,
      "German": 1.08,
      "English": 0.33,
      "Spanish": 0.49,
      "French": 2.26,
      "Hindi": 5.08,
      "Italian": 0.55,
      "Dutch": 0.83,
      "Portuguese": 1.15
    }
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "voxtral-tts-minimax9-speaker",
    "score": 0.5377777778,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Derived unweighted mean of nine language rows in Table 3; component language results are not also imported. Vendor-run comparison.",
    "raw_value": 0.5377777777777778,
    "scale_divisor": 1,
    "components": {
      "Arabic": 0.539,
      "German": 0.489,
      "English": 0.489,
      "Spanish": 0.541,
      "French": 0.378,
      "Hindi": 0.679,
      "Italian": 0.485,
      "Dutch": 0.598,
      "Portuguese": 0.642
    }
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "voxtral-tts-seed-reported-wer",
    "score": 0.0086,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 0.86,
    "scale_divisor": 100
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "voxtral-tts-seed-reported-speaker",
    "score": 0.413,
    "source": "https://arxiv.org/html/2603.25551v2",
    "source_id": "speech-voxtral-tts-report",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Table 3, Seed TTS row. Exact split not specified in report. Vendor-run comparison.",
    "raw_value": 0.413,
    "scale_divisor": 1
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0099,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 1, test-en; Fish Audio S2. Self-reported Fish Audio comparison.",
    "raw_value": 0.99,
    "scale_divisor": 100
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0054,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 1, test-zh; Fish Audio S2. Chinese character metric under source WER heading.",
    "raw_value": 0.54,
    "scale_divisor": 100
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "fish-long-tts-en-wer",
    "score": 0.0438,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 4, English WER; Fish Audio S2. See section 6.1.3 for modified length distribution.",
    "raw_value": 4.38,
    "scale_divisor": 100
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "fish-long-tts-zh-cer",
    "score": 0.0595,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 4, Chinese CER; Fish Audio S2. See section 6.1.3 for modified length distribution.",
    "raw_value": 5.95,
    "scale_divisor": 100
  },
  {
    "model": "openaudio-s1",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0107,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 1, test-en; Fish Audio S1. Self-reported Fish Audio comparison.",
    "raw_value": 1.07,
    "scale_divisor": 100
  },
  {
    "model": "openaudio-s1",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0054,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 1, test-zh; Fish Audio S1. Chinese character metric under source WER heading.",
    "raw_value": 0.54,
    "scale_divisor": 100
  },
  {
    "model": "openaudio-s1",
    "bench": "fish-long-tts-en-wer",
    "score": 0.0626,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 4, English WER; Fish Audio S1. See section 6.1.3 for modified length distribution.",
    "raw_value": 6.26,
    "scale_divisor": 100
  },
  {
    "model": "openaudio-s1",
    "bench": "fish-long-tts-zh-cer",
    "score": 0.0644,
    "source": "https://arxiv.org/html/2603.08823v2",
    "source_id": "speech-fish-s2-report",
    "by": "lab",
    "date": "2026-03-11",
    "note": "Table 4, Chinese CER; Fish Audio S1. See section 6.1.3 for modified length distribution.",
    "raw_value": 6.44,
    "scale_divisor": 100
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.013,
    "source": "https://arxiv.org/html/2606.07080v2",
    "source_id": "speech-dots-tts-report",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2, dots.tts (SOAR), 2B. Own measurement: float32, 10 Euler solver steps, CFG 1.2, effective NFE20, torch.compile disabled.",
    "raw_value": 1.3,
    "scale_divisor": 100
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0094,
    "source": "https://arxiv.org/html/2606.07080v2",
    "source_id": "speech-dots-tts-report",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2, dots.tts (SOAR), 2B. Own measurement: float32, 10 Euler solver steps, CFG 1.2, effective NFE20, torch.compile disabled. Chinese CER displayed as WER for table compatibility.",
    "raw_value": 0.94,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "voxtral-rt-2026-en-short",
    "score": 0.0727,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row Voxtral Mini Transcribe V2; Voxtral Mini Transcribe V2 offline API; explicit report row. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.27,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "voxtral-rt-2026-en-long",
    "score": 0.0711,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row Voxtral Mini Transcribe V2; Voxtral Mini Transcribe V2 offline API; explicit report row. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.11,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "voxtral-rt-2026-fleurs13",
    "score": 0.059,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row Voxtral Mini Transcribe V2; Voxtral Mini Transcribe V2 offline API; explicit report row. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 5.9,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "voxtral-rt-2026-mcv12",
    "score": 0.0807,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row Voxtral Mini Transcribe V2; Voxtral Mini Transcribe V2 offline API; explicit report row. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 8.07,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "voxtral-rt-2026-en-short",
    "score": 0.0793,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row GPT-4o mini Transcribe; GPT-4o mini Transcribe via Realtime API; distinct from full GPT-4o Transcribe. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.93,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "voxtral-rt-2026-en-long",
    "score": 0.0797,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row GPT-4o mini Transcribe; GPT-4o mini Transcribe via Realtime API; distinct from full GPT-4o Transcribe. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.97,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "voxtral-rt-2026-fleurs13",
    "score": 0.0795,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row GPT-4o mini Transcribe; GPT-4o mini Transcribe via Realtime API; distinct from full GPT-4o Transcribe. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.95,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "voxtral-rt-2026-mcv12",
    "score": 0.1285,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row GPT-4o mini Transcribe; GPT-4o mini Transcribe via Realtime API; distinct from full GPT-4o Transcribe. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 12.85,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "voxtral-rt-2026-en-short",
    "score": 0.0847,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row 480; mistralai/Voxtral-Mini-4B-Realtime-2602; target delay 480 ms only. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 8.47,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "voxtral-rt-2026-en-long",
    "score": 0.0773,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row 480; mistralai/Voxtral-Mini-4B-Realtime-2602; target delay 480 ms only. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.73,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "voxtral-rt-2026-fleurs13",
    "score": 0.0872,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row 480; mistralai/Voxtral-Mini-4B-Realtime-2602; target delay 480 ms only. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 8.72,
    "scale_divisor": 100
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "voxtral-rt-2026-mcv12",
    "score": 0.1524,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "lab",
    "date": "2026-02-11",
    "note": "Table 3 row 480; mistralai/Voxtral-Mini-4B-Realtime-2602; target delay 480 ms only. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 15.24,
    "scale_divisor": 100
  },
  {
    "model": "kyutai-stt-2-6b-en",
    "bench": "voxtral-rt-2026-en-short",
    "score": 0.0811,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row DSM 2.6B En; Kyutai DSM 2.6B English STT; target delay 2500ms. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 8.11,
    "scale_divisor": 100
  },
  {
    "model": "kyutai-stt-2-6b-en",
    "bench": "voxtral-rt-2026-en-long",
    "score": 0.0772,
    "source": "https://arxiv.org/html/2602.11298v1",
    "source_id": "speech-voxtral-realtime-report",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Table 3 row DSM 2.6B En; Kyutai DSM 2.6B English STT; target delay 2500ms. No additional delay configurations or suite subcomponents imported.",
    "raw_value": 7.72,
    "scale_divisor": 100
  },
  {
    "model": "google-chirp-3",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.3822857143,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row Chirp 3. Selected components are not separately imported.",
    "raw_value": 38.22857142857143,
    "scale_divisor": 100,
    "components": {
      "IRQ": 35.71,
      "DZA": 53.11,
      "ARE": 42.88,
      "EGY": 42.71,
      "MAR": 52.3,
      "SAU": 16.76,
      "SYR": 24.13
    }
  },
  {
    "model": "google-chirp-3",
    "bench": "gigaspeechbench-en-accents6-macro",
    "score": 0.1723333333,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 5: English accent benchmark results, reported in WER (%). Source row Chirp 3. Selected components are not separately imported.",
    "raw_value": 17.233333333333334,
    "scale_divisor": 100,
    "components": {
      "CHN-EN": 16.76,
      "IDN-EN": 8.11,
      "JPN-EN": 18.69,
      "PHL-EN": 11.96,
      "SCT-EN": 31.32,
      "SGP-EN": 16.56
    }
  },
  {
    "model": "google-chirp-3",
    "bench": "gigaspeechbench-en-domain12-bwer",
    "score": 0.146875,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 8: English vertical-domain benchmark results, reported in B-WER (%). Source row Chirp 3. Selected components are not separately imported.",
    "raw_value": 14.6875,
    "scale_divisor": 100,
    "components": {
      "AGR-EN": 7.92,
      "AIT-EN": 27.63,
      "ART-EN": 8.58,
      "BIO-EN": 15.12,
      "ECM-EN": 16.13,
      "ENG-EN": 13.28,
      "ENT-EN": 21.16,
      "FIN-EN": 12.05,
      "HUM-EN": 6.25,
      "LAW-EN": 15.97,
      "MED-EN": 13.1,
      "MIL-EN": 19.06
    }
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.4110857143,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row ElevenLabs Scribe v2. Selected components are not separately imported.",
    "raw_value": 41.10857142857143,
    "scale_divisor": 100,
    "components": {
      "IRQ": 38.67,
      "DZA": 50.43,
      "ARE": 46.1,
      "EGY": 44.44,
      "MAR": 60.06,
      "SAU": 33.33,
      "SYR": 14.73
    }
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "gigaspeechbench-en-accents6-macro",
    "score": 0.21895,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 5: English accent benchmark results, reported in WER (%). Source row ElevenLabs Scribe v2. Selected components are not separately imported.",
    "raw_value": 21.895,
    "scale_divisor": 100,
    "components": {
      "CHN-EN": 18.69,
      "IDN-EN": 11.76,
      "JPN-EN": 25.6,
      "PHL-EN": 20.13,
      "SCT-EN": 36.44,
      "SGP-EN": 18.75
    }
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "gigaspeechbench-en-domain12-bwer",
    "score": 0.1615916667,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 8: English vertical-domain benchmark results, reported in B-WER (%). Source row ElevenLabs Scribe v2. Selected components are not separately imported.",
    "raw_value": 16.159166666666668,
    "scale_divisor": 100,
    "components": {
      "AGR-EN": 9.54,
      "AIT-EN": 29.95,
      "ART-EN": 9.46,
      "BIO-EN": 16.46,
      "ECM-EN": 18.31,
      "ENG-EN": 13.46,
      "ENT-EN": 20.32,
      "FIN-EN": 14.33,
      "HUM-EN": 8.56,
      "LAW-EN": 17.18,
      "MED-EN": 14.19,
      "MIL-EN": 22.15
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.4830714286,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row Qwen3-ASR-1.7B. Selected components are not separately imported.",
    "raw_value": 48.30714285714286,
    "scale_divisor": 100,
    "components": {
      "IRQ": 41.27,
      "DZA": 63.43,
      "ARE": 53.22,
      "EGY": 59.23,
      "MAR": 76.65,
      "SAU": 25.85,
      "SYR": 18.5
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "gigaspeechbench-en-accents6-macro",
    "score": 0.1512,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 5: English accent benchmark results, reported in WER (%). Source row Qwen3-ASR-1.7B. Selected components are not separately imported.",
    "raw_value": 15.12,
    "scale_divisor": 100,
    "components": {
      "CHN-EN": 14.62,
      "IDN-EN": 7.04,
      "JPN-EN": 21.52,
      "PHL-EN": 10.81,
      "SCT-EN": 24.29,
      "SGP-EN": 12.44
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "gigaspeechbench-en-domain12-bwer",
    "score": 0.1381333333,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 8: English vertical-domain benchmark results, reported in B-WER (%). Source row Qwen3-ASR-1.7B. Selected components are not separately imported.",
    "raw_value": 13.813333333333333,
    "scale_divisor": 100,
    "components": {
      "AGR-EN": 7.36,
      "AIT-EN": 26.79,
      "ART-EN": 8.2,
      "BIO-EN": 15.67,
      "ECM-EN": 15.71,
      "ENG-EN": 11.7,
      "ENT-EN": 19.42,
      "FIN-EN": 10.35,
      "HUM-EN": 6.12,
      "LAW-EN": 15.13,
      "MED-EN": 13.52,
      "MIL-EN": 15.79
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.5049571429,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row GPT-4o Transcribe. Selected components are not separately imported.",
    "raw_value": 50.495714285714286,
    "scale_divisor": 100,
    "components": {
      "IRQ": 54.53,
      "DZA": 63.14,
      "ARE": 26.26,
      "EGY": 64.23,
      "MAR": 71.26,
      "SAU": 42.38,
      "SYR": 31.67
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "gigaspeechbench-en-accents6-macro",
    "score": 0.3987833333,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 5: English accent benchmark results, reported in WER (%). Source row GPT-4o Transcribe. Selected components are not separately imported.",
    "raw_value": 39.87833333333334,
    "scale_divisor": 100,
    "components": {
      "CHN-EN": 66.12,
      "IDN-EN": 17.12,
      "JPN-EN": 38.57,
      "PHL-EN": 38.04,
      "SCT-EN": 43.57,
      "SGP-EN": 35.85
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "gigaspeechbench-en-domain12-bwer",
    "score": 0.2479583333,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 8: English vertical-domain benchmark results, reported in B-WER (%). Source row GPT-4o Transcribe. Selected components are not separately imported.",
    "raw_value": 24.795833333333334,
    "scale_divisor": 100,
    "components": {
      "AGR-EN": 15.06,
      "AIT-EN": 44.01,
      "ART-EN": 18.79,
      "BIO-EN": 17.09,
      "ECM-EN": 36.64,
      "ENG-EN": 29.04,
      "ENT-EN": 29.87,
      "FIN-EN": 19.75,
      "HUM-EN": 10.48,
      "LAW-EN": 20.85,
      "MED-EN": 32.1,
      "MIL-EN": 23.87
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.5786428571,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row Whisper Large v3. Selected components are not separately imported.",
    "raw_value": 57.864285714285714,
    "scale_divisor": 100,
    "components": {
      "IRQ": 51.04,
      "DZA": 72.02,
      "ARE": 68.41,
      "EGY": 69.78,
      "MAR": 91.89,
      "SAU": 32.79,
      "SYR": 19.12
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeechbench-en-accents6-macro",
    "score": 0.1628,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 5: English accent benchmark results, reported in WER (%). Source row Whisper Large v3. Selected components are not separately imported.",
    "raw_value": 16.28,
    "scale_divisor": 100,
    "components": {
      "CHN-EN": 17.28,
      "IDN-EN": 7.91,
      "JPN-EN": 17.55,
      "PHL-EN": 13.68,
      "SCT-EN": 27.24,
      "SGP-EN": 14.02
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeechbench-en-domain12-bwer",
    "score": 0.1466666667,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 8: English vertical-domain benchmark results, reported in B-WER (%). Source row Whisper Large v3. Selected components are not separately imported.",
    "raw_value": 14.666666666666666,
    "scale_divisor": 100,
    "components": {
      "AGR-EN": 7.6,
      "AIT-EN": 28.53,
      "ART-EN": 9,
      "BIO-EN": 16.34,
      "ECM-EN": 16.26,
      "ENG-EN": 12.64,
      "ENT-EN": 19.47,
      "FIN-EN": 11.64,
      "HUM-EN": 6.11,
      "LAW-EN": 15.64,
      "MED-EN": 13.78,
      "MIL-EN": 18.99
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "gigaspeechbench-arabic7-macro",
    "score": 0.4655714286,
    "source": "https://arxiv.org/html/2606.28884v1",
    "source_id": "speech-gigaspeechbench-report",
    "by": "independent",
    "date": "2026-06-27",
    "note": "Derived arithmetic mean from Table 2: Low-resource languages benchmark results. WER (%) is reported for Arabic and Southeast Asian languages, while CER (%) is reported for East Asian languages. Source row Deepgram Nova 3. Selected components are not separately imported.",
    "raw_value": 46.55714285714286,
    "scale_divisor": 100,
    "components": {
      "IRQ": 47.54,
      "DZA": 57.9,
      "ARE": 52.06,
      "EGY": 52.77,
      "MAR": 60,
      "SAU": 25.02,
      "SYR": 30.61
    }
  },
  {
    "model": "deepgram-nova-2",
    "bench": "stt-ru-vakovalskii-fleurs",
    "score": 0.0397,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 775 successful utterances. Direct model identity nova-2.",
    "raw_value": 3.97,
    "scale_divisor": 100,
    "ci95_percent": [
      3.54,
      4.42
    ],
    "sample_count": 775,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/fleurs/nova-2.jsonl",
    "verification": {
      "word_edits": 600,
      "reference_words": 15098,
      "unrounded_percent": 3.9740362961981717
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "deepgram-nova-2",
    "bench": "stt-ru-vakovalskii-rulibrispeech",
    "score": 0.1022,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 295 successful utterances. Direct model identity nova-2.",
    "raw_value": 10.22,
    "scale_divisor": 100,
    "ci95_percent": [
      8.96,
      11.58
    ],
    "sample_count": 295,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/rulibrispeech/nova-2.jsonl",
    "verification": {
      "word_edits": 407,
      "reference_words": 3984,
      "unrounded_percent": 10.21586345381526
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "deepgram-nova-2",
    "bench": "stt-ru-vakovalskii-podlodka",
    "score": 0.0927,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 20 successful utterances. Direct model identity nova-2.",
    "raw_value": 9.27,
    "scale_divisor": 100,
    "ci95_percent": [
      6.7,
      12.08
    ],
    "sample_count": 20,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/podlodka/nova-2.jsonl",
    "verification": {
      "word_edits": 108,
      "reference_words": 1165,
      "unrounded_percent": 9.27038626609442
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "stt-ru-vakovalskii-fleurs",
    "score": 0.0655,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 775 successful utterances. Direct model identity nova-3.",
    "raw_value": 6.55,
    "scale_divisor": 100,
    "ci95_percent": [
      5.84,
      7.3
    ],
    "sample_count": 775,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/fleurs/nova-3.jsonl",
    "verification": {
      "word_edits": 989,
      "reference_words": 15098,
      "unrounded_percent": 6.550536494899986
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "stt-ru-vakovalskii-rulibrispeech",
    "score": 0.0888,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 296 successful utterances. Direct model identity nova-3.",
    "raw_value": 8.88,
    "scale_divisor": 100,
    "ci95_percent": [
      7.88,
      9.92
    ],
    "sample_count": 296,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/rulibrispeech/nova-3.jsonl",
    "verification": {
      "word_edits": 356,
      "reference_words": 4009,
      "unrounded_percent": 8.880019955101023
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "stt-ru-vakovalskii-podlodka",
    "score": 0.0712,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 20 successful utterances. Direct model identity nova-3.",
    "raw_value": 7.12,
    "scale_divisor": 100,
    "ci95_percent": [
      4.77,
      9.83
    ],
    "sample_count": 20,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/podlodka/nova-3.jsonl",
    "verification": {
      "word_edits": 83,
      "reference_words": 1165,
      "unrounded_percent": 7.124463519313305
    },
    "api_configuration": {
      "language": "ru",
      "punctuate": true,
      "smart_format": false
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "stt-ru-vakovalskii-fleurs",
    "score": 0.0556,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 775 successful utterances. Direct model identity parakeet-tdt-0.6b-v3.",
    "raw_value": 5.56,
    "scale_divisor": 100,
    "ci95_percent": [
      5.08,
      6.07
    ],
    "sample_count": 775,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/fleurs/parakeet-tdt-0.6b-v3.jsonl",
    "verification": {
      "word_edits": 839,
      "reference_words": 15098,
      "unrounded_percent": 5.557027420850444
    },
    "api_configuration": {
      "checkpoint": "nvidia/parakeet-tdt-0.6b-v3",
      "hardware": "RTX4090"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "stt-ru-vakovalskii-rulibrispeech",
    "score": 0.1084,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 296 successful utterances. Direct model identity parakeet-tdt-0.6b-v3.",
    "raw_value": 10.84,
    "scale_divisor": 100,
    "ci95_percent": [
      9.75,
      12
    ],
    "sample_count": 296,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/rulibrispeech/parakeet-tdt-0.6b-v3.jsonl",
    "verification": {
      "word_edits": 434,
      "reference_words": 4004,
      "unrounded_percent": 10.839160839160838
    },
    "api_configuration": {
      "checkpoint": "nvidia/parakeet-tdt-0.6b-v3",
      "hardware": "RTX4090"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "stt-ru-vakovalskii-podlodka",
    "score": 0.0927,
    "source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md",
    "source_id": "speech-stt-ru-mit",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Corpus WER independently recomputed from pinned raw JSONL, matching published two-decimal value. 20 successful utterances. Direct model identity parakeet-tdt-0.6b-v3.",
    "raw_value": 9.27,
    "scale_divisor": 100,
    "ci95_percent": [
      7.16,
      11.72
    ],
    "sample_count": 20,
    "raw_source": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/results/podlodka/parakeet-tdt-0.6b-v3.jsonl",
    "verification": {
      "word_edits": 108,
      "reference_words": 1165,
      "unrounded_percent": 9.27038626609442
    },
    "api_configuration": {
      "checkpoint": "nvidia/parakeet-tdt-0.6b-v3",
      "hardware": "RTX4090"
    }
  },
  {
    "model": "deepgram-nova-2",
    "bench": "memo20-live-ru-full-median",
    "score": 0.2143,
    "source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv",
    "source_id": "speech-smolevich-memo-licensed",
    "by": "independent",
    "date": "2026-08-03",
    "note": "Pinned summary.csv full live ru group; median reproduced from 30 committed utterance-run WER values. Three runs of each of ten clips. Small pilot only.",
    "raw_value": 21.43,
    "scale_divisor": 100,
    "run_count": 30,
    "sample_count": 10,
    "raw_source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/consolidated.json",
    "source_run_files": [
      "stt_landscape_20260803T231434Z.json"
    ],
    "configuration": {
      "model": "nova-2",
      "language": "ru",
      "smart_format": true,
      "punctuate": true
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "memo20-live-ru-full-median",
    "score": 0.3101,
    "source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv",
    "source_id": "speech-smolevich-memo-licensed",
    "by": "independent",
    "date": "2026-08-03",
    "note": "Pinned summary.csv full live ru group; median reproduced from 30 committed utterance-run WER values. Three runs of each of ten clips. Small pilot only.",
    "raw_value": 31.01,
    "scale_divisor": 100,
    "run_count": 30,
    "sample_count": 10,
    "raw_source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/consolidated.json",
    "source_run_files": [
      "stt_landscape_20260803T231434Z.json"
    ],
    "configuration": {
      "model": "nova-3",
      "language": "ru",
      "smart_format": true,
      "punctuate": true
    }
  },
  {
    "model": "deepgram-nova-2",
    "bench": "memo20-live-en-full-median",
    "score": 0.2047,
    "source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv",
    "source_id": "speech-smolevich-memo-licensed",
    "by": "independent",
    "date": "2026-08-03",
    "note": "Pinned summary.csv full live en group; median reproduced from 30 committed utterance-run WER values. Three runs of each of ten clips. Small pilot only.",
    "raw_value": 20.47,
    "scale_divisor": 100,
    "run_count": 30,
    "sample_count": 10,
    "raw_source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/consolidated.json",
    "source_run_files": [
      "stt_landscape_20260803T231434Z.json"
    ],
    "configuration": {
      "model": "nova-2",
      "language": "en",
      "smart_format": true,
      "punctuate": true
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "memo20-live-en-full-median",
    "score": 0.2361,
    "source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv",
    "source_id": "speech-smolevich-memo-licensed",
    "by": "independent",
    "date": "2026-08-03",
    "note": "Pinned summary.csv full live en group; median reproduced from 30 committed utterance-run WER values. Three runs of each of ten clips. Small pilot only.",
    "raw_value": 23.61,
    "scale_divisor": 100,
    "run_count": 30,
    "sample_count": 10,
    "raw_source": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/consolidated.json",
    "source_run_files": [
      "stt_landscape_20260803T231434Z.json"
    ],
    "configuration": {
      "model": "nova-3",
      "language": "en",
      "smart_format": true,
      "punctuate": true
    }
  },
  {
    "model": "seed-tts-2-0-standard",
    "bench": "cn-newstts-v01-strict",
    "score": 0.879032,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model seed-tts-2.0-standard, voice configured default Chinese voice. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.879032,
    "scale_divisor": 1,
    "counts": {
      "correct": 872,
      "wrong": 34,
      "unknown": 86,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "Volcano / Doubao TTS",
      "name_zh": "火山 / 豆包 TTS",
      "provider_en": "ByteDance Volcengine",
      "provider_zh": "字节火山引擎",
      "api_source_en": "Volcengine Speech Synthesis API",
      "api_source_zh": "火山引擎语音合成 API",
      "model": "seed-tts-2.0-standard",
      "voice": "configured default Chinese voice",
      "sample_rate": 24000,
      "audio_format": "wav"
    }
  },
  {
    "model": "azure-neural-tts-xiaoxiao",
    "bench": "cn-newstts-v01-strict",
    "score": 0.756048,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model azure-neural-tts, voice zh-CN-XiaoxiaoNeural. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.756048,
    "scale_divisor": 1,
    "counts": {
      "correct": 750,
      "wrong": 0,
      "unknown": 242,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "Azure Speech TTS",
      "name_zh": "Azure Speech TTS",
      "provider_en": "Microsoft Azure",
      "provider_zh": "Microsoft Azure",
      "api_source_en": "Azure AI Speech Text-to-Speech",
      "api_source_zh": "Azure AI Speech 文本转语音",
      "model": "azure-neural-tts",
      "voice": "zh-CN-XiaoxiaoNeural",
      "sample_rate": 24000,
      "audio_format": "wav"
    }
  },
  {
    "model": "google-chirp-3-hd",
    "bench": "cn-newstts-v01-strict",
    "score": 0.603831,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model Chirp 3 HD, voice cmn-CN-Chirp3-HD-Kore. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.603831,
    "scale_divisor": 1,
    "counts": {
      "correct": 599,
      "wrong": 255,
      "unknown": 138,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "Google Cloud TTS",
      "name_zh": "Google Cloud TTS",
      "provider_en": "Google Cloud",
      "provider_zh": "Google Cloud",
      "api_source_en": "Cloud Text-to-Speech API",
      "api_source_zh": "Cloud Text-to-Speech API",
      "model": "Chirp 3 HD",
      "voice": "cmn-CN-Chirp3-HD-Kore",
      "sample_rate": 24000,
      "audio_format": "wav"
    }
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cn-newstts-v01-strict",
    "score": 0.548387,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model speech-2.8-hd, voice configured Mandarin news voice. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.548387,
    "scale_divisor": 1,
    "counts": {
      "correct": 544,
      "wrong": 299,
      "unknown": 149,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "MiniMax TTS",
      "name_zh": "MiniMax TTS",
      "provider_en": "MiniMax",
      "provider_zh": "MiniMax",
      "api_source_en": "MiniMax Speech API",
      "api_source_zh": "MiniMax Speech API",
      "model": "speech-2.8-hd",
      "voice": "configured Mandarin news voice",
      "sample_rate": 24000,
      "audio_format": "mp3"
    }
  },
  {
    "model": "cosyvoice-v3-plus",
    "bench": "cn-newstts-v01-strict",
    "score": 0.471774,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model cosyvoice-v3-plus, voice configured default Chinese voice. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.471774,
    "scale_divisor": 1,
    "counts": {
      "correct": 468,
      "wrong": 61,
      "unknown": 463,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "Aliyun CosyVoice",
      "name_zh": "阿里云 CosyVoice",
      "provider_en": "Alibaba Cloud",
      "provider_zh": "阿里云",
      "api_source_en": "DashScope / Model Studio Speech API",
      "api_source_zh": "DashScope / 百炼语音 API",
      "model": "cosyvoice-v3-plus",
      "voice": "configured default Chinese voice",
      "sample_rate": 24000,
      "audio_format": "wav"
    }
  },
  {
    "model": "mimo-v2-5-tts",
    "bench": "cn-newstts-v01-strict",
    "score": 0.275202,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model mimo-v2.5-tts, voice 白桦. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.275202,
    "scale_divisor": 1,
    "counts": {
      "correct": 273,
      "wrong": 350,
      "unknown": 369,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "MiMo TTS",
      "name_zh": "MiMo TTS",
      "provider_en": "Xiaomi MiMo",
      "provider_zh": "小米 MiMo",
      "api_source_en": "MiMo TTS API",
      "api_source_zh": "MiMo TTS API",
      "model": "mimo-v2.5-tts",
      "voice": "白桦",
      "sample_rate": 24000,
      "audio_format": "wav"
    }
  },
  {
    "model": "amazon-polly-neural",
    "bench": "cn-newstts-v01-strict",
    "score": 0.243952,
    "source": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv",
    "source_id": "speech-cn-news-tts-ccby",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Primary strict score: correct/992; unknown targets remain in denominator. Source model polly-neural, voice Zhiyu. No resolved-accuracy/coverage duplicate results.",
    "raw_value": 0.243952,
    "scale_divisor": 1,
    "counts": {
      "correct": 242,
      "wrong": 323,
      "unknown": 427,
      "total_auto_evaluable_targets": 992,
      "optional_targets_excluded": 16
    },
    "configuration": {
      "name_en": "AWS Polly",
      "name_zh": "AWS Polly",
      "provider_en": "Amazon Web Services",
      "provider_zh": "Amazon Web Services",
      "api_source_en": "Amazon Polly",
      "api_source_zh": "Amazon Polly",
      "model": "polly-neural",
      "voice": "Zhiyu",
      "sample_rate": 24000,
      "audio_format": "mp3"
    }
  },
  {
    "model": "mage-flow-base",
    "bench": "geneval",
    "score": 0.79,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.79",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:geneval"
  },
  {
    "model": "mage-flow-base",
    "bench": "dpg-bench",
    "score": 0.8626,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "86.26",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:dpg-bench"
  },
  {
    "model": "mage-flow-base",
    "bench": "tiif-testmini-short",
    "score": 0.825,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "82.50",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-short"
  },
  {
    "model": "mage-flow-base",
    "bench": "tiif-testmini-long",
    "score": 0.8319,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "83.19",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-long"
  },
  {
    "model": "mage-flow-base",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.851,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.851",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:cvtg-2k-word-accuracy"
  },
  {
    "model": "mage-flow-base",
    "bench": "oneig-en",
    "score": 0.542,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.542",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-en"
  },
  {
    "model": "mage-flow-base",
    "bench": "oneig-cn",
    "score": 0.509,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.509",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-cn"
  },
  {
    "model": "mage-flow-base",
    "bench": "longtext-en",
    "score": 0.904,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.904",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-en"
  },
  {
    "model": "mage-flow-base",
    "bench": "longtext-cn",
    "score": 0.792,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Base; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Base",
      "table": "S6.T8",
      "reported_value": "0.792",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 30 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Base",
        "4B",
        "30",
        "0.79",
        "86.26",
        "82.50",
        "83.19",
        "0.851",
        "0.542",
        "0.509",
        "0.904",
        "0.792"
      ],
      "steps": "30",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-cn"
  },
  {
    "model": "mage-flow",
    "bench": "geneval",
    "score": 0.9,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.90",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:geneval"
  },
  {
    "model": "mage-flow",
    "bench": "dpg-bench",
    "score": 0.8649,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "86.49",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:dpg-bench"
  },
  {
    "model": "mage-flow",
    "bench": "tiif-testmini-short",
    "score": 0.8219,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "82.19",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-short"
  },
  {
    "model": "mage-flow",
    "bench": "tiif-testmini-long",
    "score": 0.847,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "84.70",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-long"
  },
  {
    "model": "mage-flow",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.887,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.887",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:cvtg-2k-word-accuracy"
  },
  {
    "model": "mage-flow",
    "bench": "oneig-en",
    "score": 0.536,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.536",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-en"
  },
  {
    "model": "mage-flow",
    "bench": "oneig-cn",
    "score": 0.505,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.505",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-cn"
  },
  {
    "model": "mage-flow",
    "bench": "longtext-en",
    "score": 0.944,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.944",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-en"
  },
  {
    "model": "mage-flow",
    "bench": "longtext-cn",
    "score": 0.823,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow",
      "table": "S6.T8",
      "reported_value": "0.823",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 20 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow",
        "4B",
        "20",
        "0.90",
        "86.49",
        "82.19",
        "84.70",
        "0.887",
        "0.536",
        "0.505",
        "0.944",
        "0.823"
      ],
      "steps": "20",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-cn"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "geneval",
    "score": 0.88,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.88",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:geneval"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "dpg-bench",
    "score": 0.8548,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "85.48",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:dpg-bench"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "tiif-testmini-short",
    "score": 0.8358,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "83.58",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-short"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "tiif-testmini-long",
    "score": 0.8416,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "84.16",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:tiif-testmini-long"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.873,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.873",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:cvtg-2k-word-accuracy"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "oneig-en",
    "score": 0.523,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.523",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-en"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "oneig-cn",
    "score": 0.491,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.491",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:oneig-cn"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "longtext-en",
    "score": 0.911,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.911",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-en"
  },
  {
    "model": "mage-flow-turbo",
    "bench": "longtext-cn",
    "score": 0.801,
    "source": "https://arxiv.org/html/2607.19064v2#S6.T8",
    "source_id": "visual-mage-flow",
    "by": "lab",
    "date": "2026-07-22",
    "note": "Mage-Flow-Turbo; S6.T8; T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
    "evidence": {
      "source_model_label": "Mage-Flow-Turbo",
      "table": "S6.T8",
      "reported_value": "0.801",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; report states native 1024² resolution and official benchmark protocols; 4 denoising steps as printed (– = not reported); prompt rewriting/CFG/API snapshot not specified per baseline.",
      "source_row": [
        "Mage-Flow-Turbo",
        "4B",
        "4",
        "0.88",
        "85.48",
        "83.58",
        "84.16",
        "0.873",
        "0.523",
        "0.491",
        "0.911",
        "0.801"
      ],
      "steps": "4",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.19064v2#S6.SS1.SSS0.Px1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Mage-Flow authors / Microsoft",
    "source_group": "visual-mage-flow:longtext-cn"
  },
  {
    "model": "hidream-o1-image",
    "bench": "geneval",
    "score": 0.9,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "source_id": "visual-hidream",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image; quantitative-evaluation; Base 50-step HiDream-O1-Image, not Dev, Dev-2604 or Pro; repository GenEval/DPG overall. Prompt-agent use not specified in evaluation table.",
    "evidence": {
      "source_model_label": "HiDream-O1-Image",
      "table": "quantitative-evaluation",
      "reported_value": "0.90",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Base 50-step HiDream-O1-Image, not Dev, Dev-2604 or Pro; repository GenEval/DPG overall. Prompt-agent use not specified in evaluation table.",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "HiDream.ai",
    "source_group": "visual-hidream:geneval"
  },
  {
    "model": "hidream-o1-image",
    "bench": "dpg-bench",
    "score": 0.8983,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "source_id": "visual-hidream",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image; quantitative-evaluation; Base 50-step HiDream-O1-Image, not Dev, Dev-2604 or Pro; repository GenEval/DPG overall. Prompt-agent use not specified in evaluation table.",
    "evidence": {
      "source_model_label": "HiDream-O1-Image",
      "table": "quantitative-evaluation",
      "reported_value": "89.83",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Base 50-step HiDream-O1-Image, not Dev, Dev-2604 or Pro; repository GenEval/DPG overall. Prompt-agent use not specified in evaluation table.",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "HiDream.ai",
    "source_group": "visual-hidream:dpg-bench"
  },
  {
    "model": "ernie-image",
    "bench": "geneval",
    "score": 0.8856,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "ERNIE-Image (w/o PE); evaluation; Without prompt enhancer (PE); no substitution with w/PE scores 0.8728/0.8510.",
    "evidence": {
      "source_model_label": "ERNIE-Image (w/o PE)",
      "table": "evaluation",
      "reported_value": "0.8856",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Without prompt enhancer (PE); no substitution with w/PE scores 0.8728/0.8510.",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Baidu / ERNIE-Image authors",
    "source_group": "visual-ernie-image:geneval"
  },
  {
    "model": "ernie-image",
    "bench": "oneig-en",
    "score": 0.5537,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "ERNIE-Image (w/o PE); evaluation; OneIG-EN; without prompt enhancer (PE).",
    "evidence": {
      "source_model_label": "ERNIE-Image (w/o PE)",
      "table": "evaluation",
      "reported_value": "0.5537",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "OneIG-EN; without prompt enhancer (PE).",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Baidu / ERNIE-Image authors",
    "source_group": "visual-ernie-image:oneig-en"
  },
  {
    "model": "ernie-image-turbo",
    "bench": "geneval",
    "score": 0.8667,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "ERNIE-Image-Turbo (w/o PE); evaluation; Without prompt enhancer (PE); no substitution with w/PE scores 0.8728/0.8510.",
    "evidence": {
      "source_model_label": "ERNIE-Image-Turbo (w/o PE)",
      "table": "evaluation",
      "reported_value": "0.8667",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Without prompt enhancer (PE); no substitution with w/PE scores 0.8728/0.8510.",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Baidu / ERNIE-Image authors",
    "source_group": "visual-ernie-image:geneval"
  },
  {
    "model": "ernie-image-turbo",
    "bench": "oneig-en",
    "score": 0.5341,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "ERNIE-Image-Turbo (w/o PE); evaluation; OneIG-EN; without prompt enhancer (PE).",
    "evidence": {
      "source_model_label": "ERNIE-Image-Turbo (w/o PE)",
      "table": "evaluation",
      "reported_value": "0.5341",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "OneIG-EN; without prompt enhancer (PE).",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "Baidu / ERNIE-Image authors",
    "source_group": "visual-ernie-image:oneig-en"
  },
  {
    "model": "glm-image",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.9116,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "0.9116",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.9116",
        "0.9557",
        "0.7877",
        "0.966",
        "0.952",
        "0.979"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:cvtg-2k-word-accuracy"
  },
  {
    "model": "glm-image",
    "bench": "longtext-en",
    "score": 0.952,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "0.952",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.9116",
        "0.9557",
        "0.7877",
        "0.966",
        "0.952",
        "0.979"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:longtext-en"
  },
  {
    "model": "glm-image",
    "bench": "longtext-cn",
    "score": 0.979,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "0.979",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README text rendering; CVTG Word Accuracy or language-specific LongText score.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.9116",
        "0.9557",
        "0.7877",
        "0.966",
        "0.952",
        "0.979"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:longtext-cn"
  },
  {
    "model": "glm-image",
    "bench": "oneig-en",
    "score": 0.528,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "0.528",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.528",
        "0.511",
        "81.01",
        "81.02",
        "84.78"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:oneig-en"
  },
  {
    "model": "glm-image",
    "bench": "oneig-cn",
    "score": 0.511,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "0.511",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.528",
        "0.511",
        "81.01",
        "81.02",
        "84.78"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:oneig-cn"
  },
  {
    "model": "glm-image",
    "bench": "dpg-bench",
    "score": 0.8478,
    "source": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
    "source_id": "speed-glm-image",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-Image; evaluation; Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "evaluation",
      "reported_value": "84.78",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Provider README general image-generation evaluation; row exact as published; TIIF testmini as referenced comparison protocol.",
      "source_row": [
        "GLM-Image",
        "✓",
        "0.528",
        "0.511",
        "81.01",
        "81.02",
        "84.78"
      ],
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/zai-org/GLM-Image/blob/69b87db2874f8b556417c03eedf2b8a1484f62e0/README.md",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Table does not establish prompt-refiner use, exact CFG or complete per-benchmark sampling settings. README usage defaults are not assumed to be evaluation settings."
    },
    "evaluator": "Z.ai / GLM-Image authors",
    "source_group": "visual-glm-image:dpg-bench"
  },
  {
    "model": "sd-1-5",
    "bench": "geneval",
    "score": 0.43,
    "source": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "note": "Stable Diffusion v1.5; main-results; Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
    "evidence": {
      "source_model_label": "Stable Diffusion v1.5",
      "table": "main-results",
      "reported_value": "0.43",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
      "reported_date_basis": "GenEval paper first publication; repository table itself has no per-row date",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "GenEval authors: Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt",
    "source_group": "visual-geneval:geneval"
  },
  {
    "model": "sd-2-1",
    "bench": "geneval",
    "score": 0.5,
    "source": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "note": "Stable Diffusion v2.1; main-results; Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
    "evidence": {
      "source_model_label": "Stable Diffusion v2.1",
      "table": "main-results",
      "reported_value": "0.50",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
      "reported_date_basis": "GenEval paper first publication; repository table itself has no per-row date",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "GenEval authors: Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt",
    "source_group": "visual-geneval:geneval"
  },
  {
    "model": "sdxl",
    "bench": "geneval",
    "score": 0.55,
    "source": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "note": "Stable Diffusion XL; main-results; Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
    "evidence": {
      "source_model_label": "Stable Diffusion XL",
      "table": "main-results",
      "reported_value": "0.55",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
      "reported_date_basis": "GenEval paper first publication; repository table itself has no per-row date",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "GenEval authors: Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt",
    "source_group": "visual-geneval:geneval"
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "geneval",
    "score": 0.61,
    "source": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "note": "IF-XL; main-results; Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
    "evidence": {
      "source_model_label": "IF-XL",
      "table": "main-results",
      "reported_value": "0.61",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original GenEval 553 prompts, 4 images per prompt; original benchmark-author run.",
      "reported_date_basis": "GenEval paper first publication; repository table itself has no per-row date",
      "date_basis": "Retrieval date unless specifically stated; source does not provide per-row measurement dates",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/djghosh13/geneval/blob/af4902f24d3ca90ebbb446dd9891a59e0f82725f/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "GenEval authors: Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt",
    "source_group": "visual-geneval:geneval"
  },
  {
    "model": "seedance-1-0-pro-fast",
    "bench": "humanscore-t2v",
    "score": 0.911,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Seedance 1.0 Pro fast; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Seedance 1.0 Pro fast",
      "table": "S3.T1",
      "reported_value": "91.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Seedance 1.0 Pro fast",
        "94.2",
        "93.6",
        "93.9",
        "83.6",
        "85.8",
        "84.7",
        "94.5",
        "94.2",
        "94.3",
        "91.1"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "kling-2-5-turbo-pro",
    "bench": "humanscore-t2v",
    "score": 0.908,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "KlingAI 2.5 Turbo Pro; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "KlingAI 2.5 Turbo Pro",
      "table": "S3.T1",
      "reported_value": "90.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "KlingAI 2.5 Turbo Pro",
        "89.3",
        "92.6",
        "91.0",
        "82.4",
        "90.3",
        "86.4",
        "95.2",
        "94.9",
        "95.1",
        "90.8"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "luma-ray-3",
    "bench": "humanscore-t2v",
    "score": 0.877,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Ray 3.0; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Ray 3.0",
      "table": "S3.T1",
      "reported_value": "87.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Ray 3.0",
        "80.5",
        "92.8",
        "86.7",
        "76.0",
        "89.0",
        "82.5",
        "93.9",
        "93.6",
        "93.8",
        "87.7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "sora-2",
    "bench": "humanscore-t2v",
    "score": 0.861,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Sora-2; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Sora-2",
      "table": "S3.T1",
      "reported_value": "86.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Sora-2",
        "91.9",
        "89.7",
        "90.8",
        "72.5",
        "83.8",
        "78.2",
        "90.9",
        "87.9",
        "89.4",
        "86.1"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "veo-3-1-fast",
    "bench": "humanscore-t2v",
    "score": 0.859,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Veo 3.1 fast; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Veo 3.1 fast",
      "table": "S3.T1",
      "reported_value": "85.9",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Veo 3.1 fast",
        "78.4",
        "90.8",
        "84.6",
        "72.0",
        "87.5",
        "79.8",
        "93.8",
        "92.8",
        "93.3",
        "85.9"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "hailuo-02",
    "bench": "humanscore-t2v",
    "score": 0.858,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Hailuo 02; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Hailuo 02",
      "table": "S3.T1",
      "reported_value": "85.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Hailuo 02",
        "85.6",
        "92.5",
        "89.1",
        "71.3",
        "82.8",
        "77.1",
        "91.9",
        "90.6",
        "91.2",
        "85.8"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "pixverse-v5-5",
    "bench": "humanscore-t2v",
    "score": 0.855,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "PixVerse 5.5; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "PixVerse 5.5",
      "table": "S3.T1",
      "reported_value": "85.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "PixVerse 5.5",
        "82.9",
        "91.0",
        "87.0",
        "71.3",
        "85.9",
        "78.6",
        "91.3",
        "90.7",
        "91.0",
        "85.5"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "wan-2-6",
    "bench": "humanscore-t2v",
    "score": 0.848,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Wan 2.6; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Wan 2.6",
      "table": "S3.T1",
      "reported_value": "84.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Wan 2.6",
        "85.8",
        "93.3",
        "89.6",
        "68.1",
        "87.9",
        "78.0",
        "88.6",
        "84.9",
        "86.8",
        "84.8"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "pika-2-2",
    "bench": "humanscore-t2v",
    "score": 0.816,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Pika v2.2; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Pika v2.2",
      "table": "S3.T1",
      "reported_value": "81.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Pika v2.2",
        "86.0",
        "90.3",
        "88.2",
        "67.0",
        "82.5",
        "74.8",
        "83.1",
        "80.6",
        "81.8",
        "81.6"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "humanscore-t2v",
    "score": 0.911,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "HunyuanVideo 1.5; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "HunyuanVideo 1.5",
      "table": "S3.T1",
      "reported_value": "91.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "HunyuanVideo 1.5",
        "95.6",
        "94.9",
        "95.3",
        "80.8",
        "85.2",
        "83.0",
        "95.1",
        "94.8",
        "94.9",
        "91.1"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "kandinsky-5-0-pro",
    "bench": "humanscore-t2v",
    "score": 0.865,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Kandinsky 5.0 pro; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "Kandinsky 5.0 pro",
      "table": "S3.T1",
      "reported_value": "86.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "Kandinsky 5.0 pro",
        "81.8",
        "91.6",
        "86.7",
        "75.7",
        "85.6",
        "80.7",
        "92.8",
        "91.4",
        "92.1",
        "86.5"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "cogvideox-5b",
    "bench": "humanscore-t2v",
    "score": 0.748,
    "source": "https://arxiv.org/html/2604.20157v1#S3.T1",
    "source_id": "visual-humanscore",
    "by": "independent",
    "date": "2026-04-22",
    "note": "CogVideoX-5B; S3.T1; Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
    "evidence": {
      "source_model_label": "CogVideoX-5B",
      "table": "S3.T1",
      "reported_value": "74.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-prompt HumanScore v1; 102 gentle/intense motion prompts; exact release/tier in Appendix G; per-model resolution/steps/API snapshot not supplied.",
      "source_row": [
        "CogVideoX-5B",
        "88.5",
        "59.1",
        "73.8",
        "58.9",
        "69.7",
        "64.3",
        "80.1",
        "92.5",
        "86.3",
        "74.8"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2604.20157v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "HumanScore authors / Stanford and Peking University",
    "source_group": "visual-humanscore:humanscore-t2v"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1366,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Gemini Omni 1.1 Flash; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Gemini Omni 1.1 Flash",
      "table": "rankings",
      "reported_value": "1366",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 39,
      "source_row": [
        "2",
        "Gemini Omni 1.1 Flash",
        "Google",
        "1366",
        "39"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "minimax-h3",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1339,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "MiniMax H3; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "MiniMax H3",
      "table": "rankings",
      "reported_value": "1339",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 38,
      "source_row": [
        "3",
        "MiniMax H3",
        "MiniMax",
        "1339",
        "38"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "gemini-omni-flash",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1286,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Gemini Omni Flash; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Gemini Omni Flash",
      "table": "rankings",
      "reported_value": "1286",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 38,
      "source_row": [
        "4",
        "Gemini Omni Flash",
        "Google",
        "1286",
        "38"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "seedance-2-0",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1268,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Dreamina Seedance 2.0; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Dreamina Seedance 2.0",
      "table": "rankings",
      "reported_value": "1268",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "5",
        "Dreamina Seedance 2.0",
        "ByteDance",
        "1268",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "flux-3-video",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1265,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "FLUX 3 Video; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "FLUX 3 Video",
      "table": "rankings",
      "reported_value": "1265",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "6",
        "FLUX 3 Video",
        "Black Forest Labs",
        "1265",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "seedance-2-5",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1247,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Dreamina Seedance 2.5; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Dreamina Seedance 2.5",
      "table": "rankings",
      "reported_value": "1247",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "7",
        "Dreamina Seedance 2.5",
        "ByteDance",
        "1247",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "wan-3-0",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1183,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Wan 3.0; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Wan 3.0",
      "table": "rankings",
      "reported_value": "1183",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "8",
        "Wan 3.0",
        "Alibaba Cloud",
        "1183",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1143,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Grok Imagine Video 1.5; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Grok Imagine Video 1.5",
      "table": "rankings",
      "reported_value": "1143",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "9",
        "Grok Imagine Video 1.5",
        "xAI",
        "1143",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "kling-3-0-pro",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1123,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Kling 3.0 Pro; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Kling 3.0 Pro",
      "table": "rankings",
      "reported_value": "1123",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "10",
        "Kling 3.0 Pro",
        "Kuaishou",
        "1123",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "grok-imagine-video",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1118,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Grok Imagine Video; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Grok Imagine Video",
      "table": "rankings",
      "reported_value": "1118",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "11",
        "Grok Imagine Video",
        "xAI",
        "1118",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "happyhorse-1-0",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1085,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "HappyHorse 1.0; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "HappyHorse 1.0",
      "table": "rankings",
      "reported_value": "1085",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "12",
        "HappyHorse 1.0",
        "Alibaba",
        "1085",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "veo-3-1",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1077,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Veo 3.1; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Veo 3.1",
      "table": "rankings",
      "reported_value": "1077",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "13",
        "Veo 3.1",
        "Google",
        "1077",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "veo-3-1-fast",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1068,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Veo 3.1 Fast; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Veo 3.1 Fast",
      "table": "rankings",
      "reported_value": "1068",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "14",
        "Veo 3.1 Fast",
        "Google",
        "1068",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "kling-2-6-pro",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1034,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Kling 2.6 Pro; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Kling 2.6 Pro",
      "table": "rankings",
      "reported_value": "1034",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "15",
        "Kling 2.6 Pro",
        "Kuaishou",
        "1034",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "wan-2-6",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1023,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Wan 2.6; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Wan 2.6",
      "table": "rankings",
      "reported_value": "1023",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "16",
        "Wan 2.6",
        "Alibaba Cloud",
        "1023",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "wan-2-7",
    "bench": "rapidata-physics-i2v-reference",
    "score": 1009,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Wan 2.7; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Wan 2.7",
      "table": "rankings",
      "reported_value": "1009",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "17",
        "Wan 2.7",
        "Alibaba Cloud",
        "1009",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "vidu-q3-pro",
    "bench": "rapidata-physics-i2v-reference",
    "score": 987,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Vidu Q3 Pro; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Vidu Q3 Pro",
      "table": "rankings",
      "reported_value": "987",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "18",
        "Vidu Q3 Pro",
        "Shengshu",
        "987",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "pixverse-v5-6",
    "bench": "rapidata-physics-i2v-reference",
    "score": 942,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Pixverse V5.6; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Pixverse V5.6",
      "table": "rankings",
      "reported_value": "942",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 36,
      "source_row": [
        "19",
        "Pixverse V5.6",
        "AIsphere",
        "942",
        "36"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "wan-2-5-preview",
    "bench": "rapidata-physics-i2v-reference",
    "score": 918,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Wan 2.5 Preview; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Wan 2.5 Preview",
      "table": "rankings",
      "reported_value": "918",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 37,
      "source_row": [
        "20",
        "Wan 2.5 Preview",
        "Alibaba Cloud",
        "918",
        "37"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "rapidata-physics-i2v-reference",
    "score": 697,
    "source": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md#rankings",
    "source_id": "visual-rapid-physics",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Seedance V1.5 Pro; rankings; I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
    "evidence": {
      "source_model_label": "Seedance V1.5 Pro",
      "table": "rankings",
      "reported_value": "697",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V with Physics-IQ switch frame + text; reference recording visible to judges; four ~5s clips per scenario; published Elo only.",
      "ci95_half_width": 39,
      "source_row": [
        "22",
        "Seedance V1.5 Pro",
        "ByteDance",
        "697",
        "39"
      ],
      "date_basis": "Dataset snapshot retrieval; generation/evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/world-model-physics/blob/cf0e0724866405b3b7fb8faae3e59b9f7569138b/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-physics:rapidata-physics-i2v-reference"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "rapidata-camera-i2v",
    "score": 1284.2,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Gemini Omni Flash 1.1; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. Endpoint permits 16:9 or 9:16 only, so some stills are cropped/padded; framing differs.",
    "evidence": {
      "source_model_label": "Gemini Omni Flash 1.1",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "1284.2",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "1",
        "Gemini Omni Flash 1.1",
        "Google",
        "commercial image-to-video",
        "1284.2"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "generation_exception": "Endpoint permits 16:9 or 9:16 only, so some stills are cropped/padded; framing differs."
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "wan-3-0",
    "bench": "rapidata-camera-i2v",
    "score": 1249.2,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Wan 3.0; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. ",
    "evidence": {
      "source_model_label": "Wan 3.0",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "1249.2",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "2",
        "Wan 3.0",
        "Alibaba Cloud",
        "commercial image-to-video",
        "1249.2"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "happyhorse-1-1",
    "bench": "rapidata-camera-i2v",
    "score": 1181.1,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Happy Horse 1.1; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. ",
    "evidence": {
      "source_model_label": "Happy Horse 1.1",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "1181.1",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "3",
        "Happy Horse 1.1",
        "Alibaba",
        "commercial image-to-video",
        "1181.1"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "kling-3-0-pro",
    "bench": "rapidata-camera-i2v",
    "score": 1062.2,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Kling V3 Pro; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. ",
    "evidence": {
      "source_model_label": "Kling V3 Pro",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "1062.2",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "6",
        "Kling V3 Pro",
        "Kuaishou",
        "commercial image-to-video",
        "1062.2"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "rapidata-camera-i2v",
    "score": 1039.3,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Grok Imagine Video 1.5; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. ",
    "evidence": {
      "source_model_label": "Grok Imagine Video 1.5",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "1039.3",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "7",
        "Grok Imagine Video 1.5",
        "xAI",
        "commercial image-to-video",
        "1039.3"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "minimax-h3",
    "bench": "rapidata-camera-i2v",
    "score": 954,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "MiniMax H3; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. Provider 2K tier, documented as an upscale of a 768p base; do not assert native 1080p.",
    "evidence": {
      "source_model_label": "MiniMax H3",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "954.0",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "10",
        "MiniMax H3",
        "MiniMax",
        "commercial image-to-video",
        "954.0"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "generation_exception": "Provider 2K tier, documented as an upscale of a 768p base; do not assert native 1080p."
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "veo-3-1",
    "bench": "rapidata-camera-i2v",
    "score": 917.9,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Veo 3.1; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. 6-second output (API supports 4/6/8 seconds), not the nominal 5 seconds.",
    "evidence": {
      "source_model_label": "Veo 3.1",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "917.9",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "11",
        "Veo 3.1",
        "Google",
        "commercial image-to-video",
        "917.9"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "generation_exception": "6-second output (API supports 4/6/8 seconds), not the nominal 5 seconds."
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "vidu-q3",
    "bench": "rapidata-camera-i2v",
    "score": 877.6,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "source_id": "visual-rapid-camera",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Vidu Q3; Movement Elo. Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo. ",
    "evidence": {
      "source_model_label": "Vidu Q3",
      "table": "overall-ranking-elo-on-the-movement-leaderboard",
      "reported_value": "877.6",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Commercial I2V; same 600 prompts and reference animations; nominal 1080p/5s with the source’s model-specific exceptions; published Movement Elo.",
      "source_row": [
        "12",
        "Vidu Q3",
        "Shengshu AI",
        "commercial image-to-video",
        "877.6"
      ],
      "date_basis": "Dataset snapshot retrieval; evaluation date not separately stated",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://huggingface.co/datasets/Rapidata/camera-movement/blob/7e12622bf6a39b07bfead20e2cede1ae056459ea/README.md",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Rapidata",
    "source_group": "visual-rapid-camera:rapidata-camera-i2v"
  },
  {
    "model": "cosmos3-nano",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.291,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Cosmos3-N; A5.T4; Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 16B; 1280×720, 24fps; I2V mode",
    "evidence": {
      "source_model_label": "Cosmos3-N",
      "table": "A5.T4",
      "reported_value": "29.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 16B; 1280×720, 24fps; I2V mode",
      "std_reported": 2.4,
      "std_score": 0.024,
      "source_row": [
        "verified",
        "bpp",
        "29.1\\pm 2.4",
        "40.4\\pm 2.3",
        "22.0\\pm 3.7",
        "24.6\\pm 2.4",
        "29.5\\pm 1.7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "grok-imagine-video",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.348,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Grok Video; A5.T4; Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 1280×720, 24fps; hosted API",
    "evidence": {
      "source_model_label": "Grok Video",
      "table": "A5.T4",
      "reported_value": "34.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 1280×720, 24fps; hosted API",
      "std_reported": 0.6,
      "std_score": 0.006,
      "source_row": [
        "verified",
        "bpp",
        "34.8\\pm 0.6",
        "52.7\\pm 0.9",
        "21.4\\pm 0.6",
        "35.7\\pm 1.0",
        "29.6\\pm 0.4"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.334,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "HunyuanV-1.5; A5.T4; Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 8.3B; 848×480, 24fps",
    "evidence": {
      "source_model_label": "HunyuanV-1.5",
      "table": "A5.T4",
      "reported_value": "33.4",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 8.3B; 848×480, 24fps",
      "std_reported": 0.8,
      "std_score": 0.008,
      "source_row": [
        "verified",
        "bpp",
        "33.4\\pm 0.8",
        "47.1\\pm 1.2",
        "26.9\\pm 1.0",
        "29.7\\pm 0.6",
        "30.0\\pm 1.0"
      ],
      "checkpoint_limitation": "Release and I2V task verified; source gives output resolution but does not identify checkpoint revision. Resolution is not used to infer checkpoint identity.",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "p-video",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.253,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "P-Video; A5.T4; Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 1280×704, 24fps",
    "evidence": {
      "source_model_label": "P-Video",
      "table": "A5.T4",
      "reported_value": "25.3",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 1280×704, 24fps",
      "std_reported": 1.8,
      "std_score": 0.018,
      "source_row": [
        "verified",
        "bpp",
        "25.3\\pm 1.8",
        "38.6\\pm 2.2",
        "16.4\\pm 2.4",
        "22.9\\pm 1.8",
        "23.3\\pm 1.1"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "sora-2",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.265,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Sora 2; Table 4 verified GT / verified metric / bpp; exact published overall; source API snapshot/date ambiguity retained.",
    "evidence": {
      "source_model_label": "Sora 2",
      "table": "A5.T4",
      "reported_value": "26.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth, verified metric, bpp; four runs; single conditioning frame + text; Sora 2, 1280×720 at 30fps. Main Table 4 value; API snapshot/date ambiguity preserved below.",
      "std_reported": 0.8,
      "std_score": 0.008,
      "source_row": [
        "verified",
        "bpp",
        "26.5\\pm 0.8",
        "37.3\\pm 0.6",
        "27.0\\pm 2.2",
        "26.9\\pm 0.7",
        "14.8\\pm 0.6"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "api_snapshot_ambiguity": "Section 4 names Sora 2 (v2025-10), while the main-run footnote discusses April 2026 performance versus October 2025. Retain generic Sora 2 identity; do not infer a stable API snapshot or rewrite the release date."
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "physics-iq-verified-i2v-bpp",
    "score": 0.322,
    "source": "https://arxiv.org/html/2606.18943v1#A5.T4",
    "source_id": "visual-physics-verified",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Wan2.2; A5.T4; Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 14B; 1280×720, 16fps",
    "evidence": {
      "source_model_label": "Wan2.2",
      "table": "A5.T4",
      "reported_value": "32.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Verified ground truth; Verified metric; bpp templates; four runs; single conditioning frame + text. 14B; 1280×720, 16fps",
      "std_reported": 0.6,
      "std_score": 0.006,
      "source_row": [
        "verified",
        "bpp",
        "32.2\\pm 0.6",
        "51.1\\pm 1.0",
        "20.5\\pm 0.7",
        "28.5\\pm 0.7",
        "28.9\\pm 0.4"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.18943v1#S4.SS0.SSS0.Px1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Physics-IQ Verified authors",
    "source_group": "visual-physics-verified:physics-iq-verified-i2v-bpp"
  },
  {
    "model": "seedance-2-0",
    "bench": "vgi-bench-i2v",
    "score": 0.51,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Sdce2.0; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Sdce2.0",
      "table": "S3.T2",
      "reported_value": "51.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "sora-2",
    "bench": "vgi-bench-i2v",
    "score": 0.367,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Sora2; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 30fps, 8s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Sora2",
      "table": "S3.T2",
      "reported_value": "36.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 30fps, 8s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "veo-3-1",
    "bench": "vgi-bench-i2v",
    "score": 0.32,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Veo3.1; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 8s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Veo3.1",
      "table": "S3.T2",
      "reported_value": "32.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 8s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "kling-3-0",
    "bench": "vgi-bench-i2v",
    "score": 0.44,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Kling3.0; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s; tier not specified, not mapped to Pro Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Kling3.0",
      "table": "S3.T2",
      "reported_value": "44.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s; tier not specified, not mapped to Pro Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "wan-2-7",
    "bench": "vgi-bench-i2v",
    "score": 0.357,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Wan2.7; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 30fps, 5/10s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Wan2.7",
      "table": "S3.T2",
      "reported_value": "35.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 30fps, 5/10s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "runway-gen-4-5",
    "bench": "vgi-bench-i2v",
    "score": 0.366,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Gen4.5; S3.T2; I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Gen4.5",
      "table": "S3.T2",
      "reported_value": "36.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. 1280×720, 24fps, 5/10s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "vgi-bench-i2v",
    "score": 0.191,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "HY1.5; S3.T2; I2V problem-solving; published Overall final score, not success rate. I2V, 1280×720, 24fps, 5/10s Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "HY1.5",
      "table": "S3.T2",
      "reported_value": "19.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V problem-solving; published Overall final score, not success rate. I2V, 1280×720, 24fps, 5/10s Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "checkpoint_limitation": "Release and I2V task verified; source gives output resolution but does not identify checkpoint revision. Resolution is not used to infer checkpoint identity.",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "gpt-image-2",
    "bench": "qwen-image-bench-v1",
    "score": 0.6469,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "GPT Image 2; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "GPT Image 2",
      "table": "S4.T2",
      "reported_value": "64.69",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "GPT Image 2",
        "58.65",
        "67.53",
        "65.85",
        "57.38",
        "75.23",
        "64.69"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "qwen-image-bench-v1",
    "score": 0.5965,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "GPT Image 1.5; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "GPT Image 1.5",
      "table": "S4.T2",
      "reported_value": "59.65",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "GPT Image 1.5",
        "55.14",
        "60.88",
        "61.72",
        "53.95",
        "66.35",
        "59.65"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "nano-banana-pro",
    "bench": "qwen-image-bench-v1",
    "score": 0.5945,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Nano Banana Pro; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Nano Banana Pro",
      "table": "S4.T2",
      "reported_value": "59.45",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Nano Banana Pro",
        "55.67",
        "60.26",
        "61.25",
        "54.07",
        "66.23",
        "59.45"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "qwen-image-2-0-pro",
    "bench": "qwen-image-bench-v1",
    "score": 0.5784,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "lab",
    "date": "2026-05-27",
    "note": "Qwen Image 2.0 Pro; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Qwen Image 2.0 Pro",
      "table": "S4.T2",
      "reported_value": "57.84",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Qwen Image 2.0 Pro",
        "54.39",
        "58.67",
        "59.28",
        "51.83",
        "64.94",
        "57.84"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "model author evaluating own model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "seedream-4-5",
    "bench": "qwen-image-bench-v1",
    "score": 0.5678,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Seedream 4.5; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Seedream 4.5",
      "table": "S4.T2",
      "reported_value": "56.78",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Seedream 4.5",
        "54.41",
        "58.72",
        "57.31",
        "51.69",
        "60.64",
        "56.78"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "seedream-4-0",
    "bench": "qwen-image-bench-v1",
    "score": 0.5621,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Seedream 4.0; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Seedream 4.0",
      "table": "S4.T2",
      "reported_value": "56.21",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Seedream 4.0",
        "54.01",
        "58.81",
        "56.64",
        "51.05",
        "58.15",
        "56.21"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "flux-2-max",
    "bench": "qwen-image-bench-v1",
    "score": 0.5533,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "FLUX 2 Max; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "FLUX 2 Max",
      "table": "S4.T2",
      "reported_value": "55.33",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "FLUX 2 Max",
        "53.64",
        "56.85",
        "57.35",
        "49.35",
        "56.50",
        "55.33"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "flux-2-pro",
    "bench": "qwen-image-bench-v1",
    "score": 0.5457,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "FLUX 2 Pro; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "FLUX 2 Pro",
      "table": "S4.T2",
      "reported_value": "54.57",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "FLUX 2 Pro",
        "52.30",
        "56.94",
        "57.01",
        "47.29",
        "56.18",
        "54.57"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "gpt-image-1",
    "bench": "qwen-image-bench-v1",
    "score": 0.5407,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "GPT Image 1; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "GPT Image 1",
      "table": "S4.T2",
      "reported_value": "54.07",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "GPT Image 1",
        "52.34",
        "55.09",
        "56.28",
        "48.14",
        "55.78",
        "54.07"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "qwen-image-2512",
    "bench": "qwen-image-bench-v1",
    "score": 0.5206,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "lab",
    "date": "2026-05-27",
    "note": "Qwen Image 2512; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Qwen Image 2512",
      "table": "S4.T2",
      "reported_value": "52.06",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Qwen Image 2512",
        "51.76",
        "54.74",
        "52.72",
        "47.00",
        "50.19",
        "52.06"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "model author evaluating own model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "imagen-4-ultra",
    "bench": "qwen-image-bench-v1",
    "score": 0.5199,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Imagen 4.0 Ultra; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Imagen 4.0 Ultra",
      "table": "S4.T2",
      "reported_value": "51.99",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Imagen 4.0 Ultra",
        "50.90",
        "54.25",
        "54.02",
        "45.59",
        "51.14",
        "51.99"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "qwen-image-bench-v1",
    "score": 0.5081,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "HunyuanImage 3.0; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "HunyuanImage 3.0",
      "table": "S4.T2",
      "reported_value": "50.81",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "HunyuanImage 3.0",
        "50.35",
        "53.57",
        "52.00",
        "44.31",
        "49.12",
        "50.81"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "imagen-4",
    "bench": "qwen-image-bench-v1",
    "score": 0.5029,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Imagen 4.0; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Imagen 4.0",
      "table": "S4.T2",
      "reported_value": "50.29",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Imagen 4.0",
        "50.16",
        "52.68",
        "51.64",
        "44.84",
        "47.94",
        "50.29"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "qwen-image",
    "bench": "qwen-image-bench-v1",
    "score": 0.4923,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "lab",
    "date": "2026-05-27",
    "note": "Qwen Image; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Qwen Image",
      "table": "S4.T2",
      "reported_value": "49.23",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Qwen Image",
        "48.44",
        "52.25",
        "50.72",
        "43.16",
        "47.30",
        "49.23"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "model author evaluating own model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "kling-image-2-1",
    "bench": "qwen-image-bench-v1",
    "score": 0.4826,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Kling Image 2.1; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "Kling Image 2.1",
      "table": "S4.T2",
      "reported_value": "48.26",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Kling Image 2.1",
        "49.11",
        "50.15",
        "49.18",
        "44.74",
        "44.67",
        "48.26"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "glm-image",
    "bench": "qwen-image-bench-v1",
    "score": 0.4819,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "GLM Image; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
    "evidence": {
      "source_model_label": "GLM Image",
      "table": "S4.T2",
      "reported_value": "48.19",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "GLM Image",
        "49.26",
        "50.64",
        "47.90",
        "44.69",
        "45.23",
        "48.19"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "vgi-bench-i2v",
    "score": 0.216,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Wan2.2-I2V-A14B identified in Appendix B.1; Table 2 Overall 21.6/100; fixed half-instance evaluation. Fixed half-instance sample; preserve source v3.",
    "evidence": {
      "source_model_label": "Wan2.2",
      "table": "S3.T2",
      "reported_value": "21.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "I2V; full video task taxonomy, fixed half of instances per task/difficulty; Gemini-3-Flash judge; Overall final score; 1280×720, 16fps, 5/10s. Source v3 evaluates a fixed half of instances in each task/difficulty, not every benchmark instance; Gemini-3-Flash judge.",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "identity_resolution": "Appendix B.1 explicitly lists Wan2.2-I2V-A14B. Not inferred from parameter count or output size.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.19583v3#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "llada-image",
    "bench": "geneval",
    "score": 0.85,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T7",
    "source_id": "visual-llada-image",
    "date": "2026-09-03",
    "note": "LLaDA-Image; S5.T7; own-model lab result; adapted row licensed CC-BY-SA-4.0 in visual-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "LLaDA-Image",
      "table": "S5.T7",
      "reported_value": "0.85",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; source own-model overall; original GenEval or DPG-Bench as explicitly named; benchmark authors treat these as supplementary compositional/prompt-following diagnostics.",
      "source_row": [
        "LLaDA-Image",
        "1.00",
        "0.98",
        "0.53",
        "0.93",
        "0.78",
        "0.84",
        "0.85"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.03796v1#S5.SS1.SSS1.Px6.p1",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Source gives the benchmark tables but does not fully specify per-benchmark seed/CFG/prompt enhancement settings. Turbo is distinct from full LLaDA-Image; do not assume every result used the same step count."
    },
    "by": "lab",
    "evaluator": "LLaDA-Image authors",
    "source_group": "visual-llada-image:geneval",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-cc-by-sa.json"
  },
  {
    "model": "llada-image-turbo",
    "bench": "geneval",
    "score": 0.82,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T7",
    "source_id": "visual-llada-image",
    "date": "2026-09-03",
    "note": "LLaDA-Image Turbo; S5.T7; own-model lab result; adapted row licensed CC-BY-SA-4.0 in visual-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "LLaDA-Image Turbo",
      "table": "S5.T7",
      "reported_value": "0.82",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; source own-model overall; original GenEval or DPG-Bench as explicitly named; benchmark authors treat these as supplementary compositional/prompt-following diagnostics.",
      "source_row": [
        "LLaDA-Image Turbo",
        "1.00",
        "0.99",
        "0.42",
        "0.93",
        "0.78",
        "0.80",
        "0.82"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.03796v1#S5.SS1.SSS1.Px6.p1",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Source gives the benchmark tables but does not fully specify per-benchmark seed/CFG/prompt enhancement settings. Turbo is distinct from full LLaDA-Image; do not assume every result used the same step count."
    },
    "by": "lab",
    "evaluator": "LLaDA-Image authors",
    "source_group": "visual-llada-image:geneval",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-cc-by-sa.json"
  },
  {
    "model": "llada-image-turbo",
    "bench": "dpg-bench",
    "score": 0.8855,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T8",
    "source_id": "visual-llada-image",
    "date": "2026-09-03",
    "note": "LLaDA-Image Turbo; S5.T8; own-model lab result; adapted row licensed CC-BY-SA-4.0 in visual-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "LLaDA-Image Turbo",
      "table": "S5.T8",
      "reported_value": "88.55",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; source own-model overall; original GenEval or DPG-Bench as explicitly named; benchmark authors treat these as supplementary compositional/prompt-following diagnostics.",
      "source_row": [
        "LLaDA-Image Turbo",
        "92.23",
        "91.86",
        "93.35",
        "91.01",
        "91.53",
        "88.55"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.03796v1#S5.SS1.SSS1.Px6.p1",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Source gives the benchmark tables but does not fully specify per-benchmark seed/CFG/prompt enhancement settings. Turbo is distinct from full LLaDA-Image; do not assume every result used the same step count."
    },
    "by": "lab",
    "evaluator": "LLaDA-Image authors",
    "source_group": "visual-llada-image:dpg-bench",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-cc-by-sa.json"
  },
  {
    "model": "llada-image",
    "bench": "dpg-bench",
    "score": 0.8748,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T8",
    "source_id": "visual-llada-image",
    "date": "2026-09-03",
    "note": "LLaDA-Image; S5.T8; own-model lab result; adapted row licensed CC-BY-SA-4.0 in visual-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "LLaDA-Image",
      "table": "S5.T8",
      "reported_value": "87.48",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "T2I; source own-model overall; original GenEval or DPG-Bench as explicitly named; benchmark authors treat these as supplementary compositional/prompt-following diagnostics.",
      "source_row": [
        "LLaDA-Image",
        "90.86",
        "92.31",
        "92.37",
        "92.55",
        "91.96",
        "87.48"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.03796v1#S5.SS1.SSS1.Px6.p1",
      "evaluator_relationship": "model author evaluating own model",
      "setup_limitation": "Source gives the benchmark tables but does not fully specify per-benchmark seed/CFG/prompt enhancement settings. Turbo is distinct from full LLaDA-Image; do not assume every result used the same step count."
    },
    "by": "lab",
    "evaluator": "LLaDA-Image authors",
    "source_group": "visual-llada-image:dpg-bench",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-cc-by-sa.json"
  },
  {
    "model": "deepseek-v3",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.489,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-V3'; model_name=deepseek-chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=48.9405684755%; website rounding=48.9%; divided by 100. deepseek-chat endpoint explicitly labelled DeepSeek-V3, linked to original DeepSeek-V3, released 2024-12-26 before V3-0324 existed. No later endpoint alias resolution used. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "DeepSeek-V3"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 48.940568475452196,
      "published_display_percent": 48.9,
      "scale_divisor": 100,
      "provenance_id": "deepseek-v3/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "deepseek-chat",
      "upstream_model_repr": "DeepSeek-V3",
      "upstream_model_style": "DeepSeekCode",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.373,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-405b-Ins-FP8'; model_name=meta-llama/Meta-Llama-3.1-405B-Instruct-FP8. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=37.2868217054%; website rounding=37.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Source checkpoint is specifically FP8, retained in settings and note. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "LLama3.1-405b-Ins-FP8"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 37.286821705426355,
      "published_display_percent": 37.3,
      "scale_divisor": 100,
      "provenance_id": "llama-3-1-405b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
      "upstream_model_repr": "LLama3.1-405b-Ins-FP8",
      "upstream_model_style": "LLaMa3",
      "reasoning_effort": null,
      "weight_precision": "FP8",
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.263,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-70b-Ins'; model_name=meta-llama/Meta-Llama-3.1-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=26.2532299742%; website rounding=26.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "LLama3.1-70b-Ins"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 26.253229974160206,
      "published_display_percent": 26.3,
      "scale_divisor": 100,
      "provenance_id": "llama-3-1-70b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
      "upstream_model_repr": "LLama3.1-70b-Ins",
      "upstream_model_style": "LLaMa3",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.161,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-8b-Ins'; model_name=meta-llama/Meta-Llama-3.1-8B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=16.149870801%; website rounding=16.1%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "LLama3.1-8b-Ins"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 16.149870801033593,
      "published_display_percent": 16.1,
      "scale_divisor": 100,
      "provenance_id": "llama-3-1-8b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
      "upstream_model_repr": "LLama3.1-8b-Ins",
      "upstream_model_style": "LLaMa3",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.318,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3.3-70b-Ins'; model_name=meta-llama/Llama-3.3-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=31.7829457364%; website rounding=31.8%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Upstream link mistakenly points to Llama 3.1; model_name and model_repr both explicitly identify Llama 3.3 and take precedence. Raw metadata retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "LLama3.3-70b-Ins"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 31.782945736434108,
      "published_display_percent": 31.8,
      "scale_divisor": 100,
      "provenance_id": "llama-3-3-70b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "meta-llama/Llama-3.3-70B-Instruct",
      "upstream_model_repr": "LLama3.3-70b-Ins",
      "upstream_model_style": "LLaMa3",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.401,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-32B'; model_name=Qwen/Qwen2.5-32B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=40.0775193798%; website rounding=40.1%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Qwen2.5-Ins-32B"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 40.07751937984496,
      "published_display_percent": 40.1,
      "scale_divisor": 100,
      "provenance_id": "qwen2-5-32b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen2.5-32B-Instruct",
      "upstream_model_repr": "Qwen2.5-Ins-32B",
      "upstream_model_style": "CodeQwen15",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.41700000000000004,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-72B'; model_name=Qwen/Qwen2.5-72B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=41.6795865633%; website rounding=41.7%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Qwen2.5-Ins-72B"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 41.679586563307495,
      "published_display_percent": 41.7,
      "scale_divisor": 100,
      "provenance_id": "qwen2-5-72b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen2.5-72B-Instruct",
      "upstream_model_repr": "Qwen2.5-Ins-72B",
      "upstream_model_style": "CodeQwen15",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.253,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-7B'; model_name=Qwen/Qwen2.5-7B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.322997416%; website rounding=25.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Qwen2.5-Ins-7B"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 25.322997416020673,
      "published_display_percent": 25.3,
      "scale_divisor": 100,
      "provenance_id": "qwen2-5-7b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen2.5-7B-Instruct",
      "upstream_model_repr": "Qwen2.5-Ins-7B",
      "upstream_model_style": "CodeQwen15",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.41600000000000004,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Coder-Ins-32B'; model_name=Qwen/Qwen2.5-Coder-32B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=41.6279069767%; website rounding=41.6%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Qwen2.5-Coder-Ins-32B"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 41.627906976744185,
      "published_display_percent": 41.6,
      "scale_divisor": 100,
      "provenance_id": "qwen2-5-coder-32b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen2.5-Coder-32B-Instruct",
      "upstream_model_repr": "Qwen2.5-Coder-Ins-32B",
      "upstream_model_style": "CodeQwen15",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.262,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Coder-Ins-7B'; model_name=Qwen/Qwen2.5-Coder-7B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=26.2273901809%; website rounding=26.2%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Qwen2.5-Coder-Ins-7B"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 26.227390180878555,
      "published_display_percent": 26.2,
      "scale_divisor": 100,
      "provenance_id": "qwen2-5-coder-7b/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen2.5-Coder-7B-Instruct",
      "upstream_model_repr": "Qwen2.5-Coder-Ins-7B",
      "upstream_model_style": "CodeQwen15",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "o3",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.758,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "source_id": "livecodebench-website-20250801",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'O3 (High)'; model_name=o3__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=75.7709251101%; website rounding=75.8%; divided by 100. Exact published endpoint/config retained (o3__high); source does not disclose a dated API snapshot. High effort selected a priori, not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "O3 (High)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 75.77092511013215,
      "published_display_percent": 75.8,
      "scale_divisor": 100,
      "provenance_id": "o3/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o3__high",
      "upstream_model_repr": "O3 (High)",
      "upstream_model_style": "OpenAIReason",
      "reasoning_effort": "high",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "o4-mini",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.802,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "source_id": "livecodebench-website-20250801",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'O4-Mini (High)'; model_name=o4-mini__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=80.1762114537%; website rounding=80.2%; divided by 100. Exact published endpoint/config retained (o4-mini__high); source does not disclose a dated API snapshot. High effort selected a priori, not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "O4-Mini (High)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 80.1762114537445,
      "published_display_percent": 80.2,
      "scale_divisor": 100,
      "provenance_id": "o4-mini/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o4-mini__high",
      "upstream_model_repr": "O4-Mini (High)",
      "upstream_model_style": "OpenAIReason",
      "reasoning_effort": "high",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.659,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "source_id": "livecodebench-website-20250801",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen3-235B-A22B'; model_name=Qwen/Qwen3-235B-A22B. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=65.859030837%; website rounding=65.9%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. April 2025 hybrid checkpoint; thinking mode/token budget is not stated by the website JSON. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Qwen3-235B-A22B"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 65.85903083700441,
      "published_display_percent": 65.9,
      "scale_divisor": 100,
      "provenance_id": "qwen3-235b-a22b/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "Qwen/Qwen3-235B-A22B",
      "upstream_model_repr": "Qwen3-235B-A22B",
      "upstream_model_style": "CodeQwenInstruct",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json"
  },
  {
    "model": "ministral-3-3b",
    "bench": "math-subset-unspecified",
    "score": 0.83,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512/blob/b35d4dfe56c142746f54dbd64f579faab2744308/README.md",
    "source_id": "card-mistralai-ministral-3-3b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Instruct-2512 card, Instruct section and named size row; release identifies Instruct as FP8. MATH Maj@1=single-answer score, already a fraction. Full MATH versus MATH-500 subset is not stated. Decoding settings not established; daily-use temperature recommendations are not evaluation logs. Reasoning and Base tables are not assigned to this identity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "Ministral 3 3B",
      "column_label": "MATH Maj@1",
      "row": [
        "Ministral 3 3B",
        "0.305",
        "56.8",
        "0.830",
        "7.83"
      ],
      "column_index": 3,
      "raw_value": "0.830",
      "scale_divisor": 1,
      "section": "Benchmark Results / Instruct",
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct",
    "provisional_reason": "Only one precisely transcribed metric selected; unspecified MATH subset retained as its own benchmark ID."
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "aime-2025",
    "score": 0.721,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 3B",
      "column_label": "AIME25",
      "row": [
        "Ministral 3 3B",
        "0.721",
        "0.775",
        "0.534",
        "0.548"
      ],
      "column_index": 1,
      "raw_value": "0.721",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "aime-2024",
    "score": 0.775,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 3B",
      "column_label": "AIME24",
      "row": [
        "Ministral 3 3B",
        "0.721",
        "0.775",
        "0.534",
        "0.548"
      ],
      "column_index": 2,
      "raw_value": "0.775",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.534,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 3B",
      "column_label": "GPQA Diamond",
      "row": [
        "Ministral 3 3B",
        "0.721",
        "0.775",
        "0.534",
        "0.548"
      ],
      "column_index": 3,
      "raw_value": "0.534",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.548,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 3B",
      "column_label": "LiveCodeBench",
      "row": [
        "Ministral 3 3B",
        "0.721",
        "0.775",
        "0.534",
        "0.548"
      ],
      "column_index": 4,
      "raw_value": "0.548",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.876,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512/blob/5b26027e7b19eeb4b7352e1fed3926375dd2cb4d/README.md",
    "source_id": "card-mistralai-ministral-3-8b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Instruct-2512 card, Instruct section and named size row; release identifies Instruct as FP8. MATH Maj@1=single-answer score, already a fraction. Full MATH versus MATH-500 subset is not stated. Decoding settings not established; daily-use temperature recommendations are not evaluation logs. Reasoning and Base tables are not assigned to this identity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "Ministral 3 8B",
      "column_label": "MATH Maj@1",
      "row": [
        "Ministral 3 8B",
        "0.509",
        "66.8",
        "0.876",
        "8.08"
      ],
      "column_index": 3,
      "raw_value": "0.876",
      "scale_divisor": 1,
      "section": "Benchmark Results / Instruct",
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct",
    "provisional_reason": "Only one precisely transcribed metric selected; unspecified MATH subset retained as its own benchmark ID."
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "aime-2025",
    "score": 0.787,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 8B",
      "column_label": "AIME25",
      "row": [
        "Ministral 3 8B",
        "0.787",
        "0.860",
        "0.668",
        "0.616"
      ],
      "column_index": 1,
      "raw_value": "0.787",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "aime-2024",
    "score": 0.86,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 8B",
      "column_label": "AIME24",
      "row": [
        "Ministral 3 8B",
        "0.787",
        "0.860",
        "0.668",
        "0.616"
      ],
      "column_index": 2,
      "raw_value": "0.860",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.668,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 8B",
      "column_label": "GPQA Diamond",
      "row": [
        "Ministral 3 8B",
        "0.787",
        "0.860",
        "0.668",
        "0.616"
      ],
      "column_index": 3,
      "raw_value": "0.668",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.616,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 8B",
      "column_label": "LiveCodeBench",
      "row": [
        "Ministral 3 8B",
        "0.787",
        "0.860",
        "0.668",
        "0.616"
      ],
      "column_index": 4,
      "raw_value": "0.616",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-14b",
    "bench": "math-subset-unspecified",
    "score": 0.904,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Instruct-2512 card, Instruct section and named size row; release identifies Instruct as FP8. MATH Maj@1=single-answer score, already a fraction. Full MATH versus MATH-500 subset is not stated. Decoding settings not established; daily-use temperature recommendations are not evaluation logs. Reasoning and Base tables are not assigned to this identity. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "Ministral 3 14B",
      "column_label": "MATH Maj@1",
      "row": [
        "Ministral 3 14B",
        "0.551",
        "68.5",
        "0.904",
        "8.49"
      ],
      "column_index": 3,
      "raw_value": "0.904",
      "scale_divisor": 1,
      "section": "Benchmark Results / Instruct",
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct",
    "provisional_reason": "Only one precisely transcribed metric selected; unspecified MATH subset retained as its own benchmark ID."
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2025",
    "score": 0.85,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/51f9210f3cd20f3452a80d5819d15dc61cc50630/README.md",
    "source_id": "card-mistralai-ministral-3-14b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 14B",
      "column_label": "AIME25",
      "row": [
        "Ministral 3 14B",
        "0.850",
        "0.898",
        "0.712",
        "0.646"
      ],
      "column_index": 1,
      "raw_value": "0.850",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2024",
    "score": 0.898,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/51f9210f3cd20f3452a80d5819d15dc61cc50630/README.md",
    "source_id": "card-mistralai-ministral-3-14b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 14B",
      "column_label": "AIME24",
      "row": [
        "Ministral 3 14B",
        "0.850",
        "0.898",
        "0.712",
        "0.646"
      ],
      "column_index": 2,
      "raw_value": "0.898",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.712,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/51f9210f3cd20f3452a80d5819d15dc61cc50630/README.md",
    "source_id": "card-mistralai-ministral-3-14b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 14B",
      "column_label": "GPQA Diamond",
      "row": [
        "Ministral 3 14B",
        "0.850",
        "0.898",
        "0.712",
        "0.646"
      ],
      "column_index": 3,
      "raw_value": "0.712",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.646,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/51f9210f3cd20f3452a80d5819d15dc61cc50630/README.md",
    "source_id": "card-mistralai-ministral-3-14b-reasoning-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact Reasoning-2512 card, Reasoning section and named size row; release identifies Reasoning as BF16. Values already fractions. Not the Instruct or Base checkpoint. LiveCodeBench version/window and per-task decoding are not specified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Ministral 3 14B",
      "column_label": "LiveCodeBench",
      "row": [
        "Ministral 3 14B",
        "0.850",
        "0.898",
        "0.712",
        "0.646"
      ],
      "column_index": 4,
      "raw_value": "0.646",
      "scale_divisor": 1,
      "section": "Benchmark Results / Reasoning",
      "checkpoint_precision": "BF16"
    },
    "mode": "reasoning"
  },
  {
    "model": "devstral-small-2",
    "bench": "swe-bench-verified",
    "score": 0.68,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Devstral Small 2 24B Instruct-2512 row; release identifies FP8. Competitor rows are explicitly based on publicly reported values and are not imported. The card recommends Mistral Vibe and lists other supported agents but does not tie this table to a specific scaffold or exact decoding configuration; neither is assumed. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Devstral Small 2",
      "column_label": "SWE Bench Verified",
      "row": [
        "Devstral Small 2",
        "24",
        "68.0%",
        "55.7%",
        "22.5%"
      ],
      "column_index": 2,
      "raw_value": "68.0%",
      "scale_divisor": 100,
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct"
  },
  {
    "model": "devstral-small-2",
    "bench": "swe-bench-multilingual",
    "score": 0.557,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Devstral Small 2 24B Instruct-2512 row; release identifies FP8. Competitor rows are explicitly based on publicly reported values and are not imported. The card recommends Mistral Vibe and lists other supported agents but does not tie this table to a specific scaffold or exact decoding configuration; neither is assumed. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Devstral Small 2",
      "column_label": "SWE Bench Multilingual",
      "row": [
        "Devstral Small 2",
        "24",
        "68.0%",
        "55.7%",
        "22.5%"
      ],
      "column_index": 3,
      "raw_value": "55.7%",
      "scale_divisor": 100,
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct"
  },
  {
    "model": "devstral-small-2",
    "bench": "terminal-bench-2",
    "score": 0.225,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own Devstral Small 2 24B Instruct-2512 row; release identifies FP8. Competitor rows are explicitly based on publicly reported values and are not imported. The card recommends Mistral Vibe and lists other supported agents but does not tie this table to a specific scaffold or exact decoding configuration; neither is assumed. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Devstral Small 2",
      "column_label": "Terminal Bench 2",
      "row": [
        "Devstral Small 2",
        "24",
        "68.0%",
        "55.7%",
        "22.5%"
      ],
      "column_index": 4,
      "raw_value": "22.5%",
      "scale_divisor": 100,
      "checkpoint_precision": "FP8"
    },
    "mode": "instruct"
  },
  {
    "model": "qwen3-coder-next",
    "bench": "swe-bench-verified",
    "score": 0.706,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-Next/blob/a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb/README.md",
    "source_id": "card-qwen-qwen3-coder-next",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own purple Qwen3-Coder-Next (Ours) printed chart label embedded in the Apache-declared release README. Card explicitly supports only non-thinking mode. SWE-Agent for SWE benchmarks; Terminus-2 JSON for Terminal-Bench2.0. Other token/decoding settings and SWE Pro split not explicitly identified by this chart. No other-model bars or ambiguous Aider bar imported. External publisher asset is archived and content-hashed, not represented as an immutable remote URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/Qwen3-Coder-Next/benchmarks.png",
      "asset_sha256": "2be7fdf646f0b0e5bb878eebc82c41514042c6b9b9cbd64f83993cdcfa1c0d90",
      "label": "SWE-Bench Verified (w/ SWE-Agent)",
      "model_label": "Qwen3-Coder-Next",
      "raw_value": "70.6",
      "scale_divisor": 100
    },
    "mode": "non-thinking"
  },
  {
    "model": "qwen3-coder-next",
    "bench": "swe-bench-multilingual",
    "score": 0.628,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-Next/blob/a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb/README.md",
    "source_id": "card-qwen-qwen3-coder-next",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own purple Qwen3-Coder-Next (Ours) printed chart label embedded in the Apache-declared release README. Card explicitly supports only non-thinking mode. SWE-Agent for SWE benchmarks; Terminus-2 JSON for Terminal-Bench2.0. Other token/decoding settings and SWE Pro split not explicitly identified by this chart. No other-model bars or ambiguous Aider bar imported. External publisher asset is archived and content-hashed, not represented as an immutable remote URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/Qwen3-Coder-Next/benchmarks.png",
      "asset_sha256": "2be7fdf646f0b0e5bb878eebc82c41514042c6b9b9cbd64f83993cdcfa1c0d90",
      "label": "SWE-Bench Multilingual (w/ SWE-Agent)",
      "model_label": "Qwen3-Coder-Next",
      "raw_value": "62.8",
      "scale_divisor": 100
    },
    "mode": "non-thinking"
  },
  {
    "model": "qwen3-coder-next",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.443,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-Next/blob/a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb/README.md",
    "source_id": "card-qwen-qwen3-coder-next",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own purple Qwen3-Coder-Next (Ours) printed chart label embedded in the Apache-declared release README. Card explicitly supports only non-thinking mode. SWE-Agent for SWE benchmarks; Terminus-2 JSON for Terminal-Bench2.0. Other token/decoding settings and SWE Pro split not explicitly identified by this chart. No other-model bars or ambiguous Aider bar imported. External publisher asset is archived and content-hashed, not represented as an immutable remote URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/Qwen3-Coder-Next/benchmarks.png",
      "asset_sha256": "2be7fdf646f0b0e5bb878eebc82c41514042c6b9b9cbd64f83993cdcfa1c0d90",
      "label": "SWE-Bench Pro (w/ SWE-Agent)",
      "model_label": "Qwen3-Coder-Next",
      "raw_value": "44.3",
      "scale_divisor": 100
    },
    "mode": "non-thinking"
  },
  {
    "model": "qwen3-coder-next",
    "bench": "terminal-bench-2",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-Next/blob/a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb/README.md",
    "source_id": "card-qwen-qwen3-coder-next",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own purple Qwen3-Coder-Next (Ours) printed chart label embedded in the Apache-declared release README. Card explicitly supports only non-thinking mode. SWE-Agent for SWE benchmarks; Terminus-2 JSON for Terminal-Bench2.0. Other token/decoding settings and SWE Pro split not explicitly identified by this chart. No other-model bars or ambiguous Aider bar imported. External publisher asset is archived and content-hashed, not represented as an immutable remote URL. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/Qwen3-Coder-Next/benchmarks.png",
      "asset_sha256": "2be7fdf646f0b0e5bb878eebc82c41514042c6b9b9cbd64f83993cdcfa1c0d90",
      "label": "Terminal-Bench 2.0 (w/ Terminus-2 json)",
      "model_label": "Qwen3-Coder-Next",
      "raw_value": "36.2",
      "scale_divisor": 100
    },
    "mode": "non-thinking"
  },
  {
    "model": "mistral-small-4",
    "bench": "aime-2025",
    "score": 0.838,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own orange Mistral Small 4 - High printed label in the pinned repository chart images/image3.png; reasoning_effort=high. LiveCodeBench version/window unspecified. Recommended temperature0.7 is not asserted as a logged evaluation setting. No AA-LCR, Collie or other-model bars imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image3.png",
      "asset_sha256": "b38c8a96b124ff831fa9e7270939123565b396485b48f3219f0387142661b735",
      "label": "AIME25 / Mistral Small 4 - High",
      "model_label": "Mistral-Small-4-119B-2603",
      "raw_value": "83.8",
      "scale_divisor": 100
    },
    "mode": "high",
    "provisional_reason": "Two selected exact-source metrics; unversioned LCB remains separate."
  },
  {
    "model": "mistral-small-4",
    "bench": "livecodebench-unversioned",
    "score": 0.636,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own orange Mistral Small 4 - High printed label in the pinned repository chart images/image3.png; reasoning_effort=high. LiveCodeBench version/window unspecified. Recommended temperature0.7 is not asserted as a logged evaluation setting. No AA-LCR, Collie or other-model bars imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "visually-reviewed-chart-label",
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image3.png",
      "asset_sha256": "b38c8a96b124ff831fa9e7270939123565b396485b48f3219f0387142661b735",
      "label": "LiveCodeBench / Mistral Small 4 - High",
      "model_label": "Mistral-Small-4-119B-2603",
      "raw_value": "63.6",
      "scale_divisor": 100
    },
    "mode": "high",
    "provisional_reason": "Two selected exact-source metrics; unversioned LCB remains separate."
  },
  {
    "model": "k2-horizon-7b",
    "bench": "hmmt-feb-2026",
    "score": 0.733,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own K2-Horizon-7B final release column. Best Practices states all reported results use high reasoning effort; temperature1.0, top_p0.95, at least32768 output tokens. Validation recipe BF16, TP1, FlashAttention3 is a serving recommendation, not an assumed separate evaluation run. HLE tool mode, SWE scaffold and SciCode main/subproblem level are not specified; retained without inference. No reference-model, AA-LCR or throughput values imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HMMT Feb 2026 Competition mathematics",
      "column_label": "K2-Horizon-7B",
      "row": [
        "HMMT Feb 2026 Competition mathematics",
        "73.3",
        "Gemma 4-12B 63.1",
        "Qwen3.5-9B 65.7",
        "Granite 4.2-8B 66.5"
      ],
      "column_index": 1,
      "raw_value": "73.3",
      "scale_divisor": 100
    },
    "mode": "high"
  },
  {
    "model": "k2-horizon-7b",
    "bench": "swe-bench-verified",
    "score": 0.706,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own K2-Horizon-7B final release column. Best Practices states all reported results use high reasoning effort; temperature1.0, top_p0.95, at least32768 output tokens. Validation recipe BF16, TP1, FlashAttention3 is a serving recommendation, not an assumed separate evaluation run. HLE tool mode, SWE scaffold and SciCode main/subproblem level are not specified; retained without inference. No reference-model, AA-LCR or throughput values imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE-bench Verified Software engineering",
      "column_label": "K2-Horizon-7B",
      "row": [
        "SWE-bench Verified Software engineering",
        "70.6",
        "Gemma 4-12B 30.6",
        "Granite 4.2-8B 47.7",
        "Qwen3.5-9B 50.8"
      ],
      "column_index": 1,
      "raw_value": "70.6",
      "scale_divisor": 100
    },
    "mode": "high"
  },
  {
    "model": "k2-horizon-7b",
    "bench": "hle",
    "score": 0.186,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own K2-Horizon-7B final release column. Best Practices states all reported results use high reasoning effort; temperature1.0, top_p0.95, at least32768 output tokens. Validation recipe BF16, TP1, FlashAttention3 is a serving recommendation, not an assumed separate evaluation run. HLE tool mode, SWE scaffold and SciCode main/subproblem level are not specified; retained without inference. No reference-model, AA-LCR or throughput values imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "HLE Expert-level reasoning",
      "column_label": "K2-Horizon-7B",
      "row": [
        "HLE Expert-level reasoning",
        "18.6",
        "Granite 4.2-8B 9.7",
        "Qwen3.5-9B 14.9",
        "Gemma 4-12B 15.7"
      ],
      "column_index": 1,
      "raw_value": "18.6",
      "scale_divisor": 100
    },
    "mode": "high"
  },
  {
    "model": "k2-horizon-7b",
    "bench": "scicode-unspecified",
    "score": 0.316,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own K2-Horizon-7B final release column. Best Practices states all reported results use high reasoning effort; temperature1.0, top_p0.95, at least32768 output tokens. Validation recipe BF16, TP1, FlashAttention3 is a serving recommendation, not an assumed separate evaluation run. HLE tool mode, SWE scaffold and SciCode main/subproblem level are not specified; retained without inference. No reference-model, AA-LCR or throughput values imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SciCode Scientific coding",
      "column_label": "K2-Horizon-7B",
      "row": [
        "SciCode Scientific coding",
        "31.6",
        "Qwen3.5-9B 27.5",
        "Mistral Small 4 28.0",
        "Granite 4.2-8B 30.4"
      ],
      "column_index": 1,
      "raw_value": "31.6",
      "scale_divisor": 100
    },
    "mode": "high"
  },
  {
    "model": "k2-horizon-7b",
    "bench": "terminal-bench-2.1",
    "score": 0.391,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Own K2-Horizon-7B final release column. Best Practices states all reported results use high reasoning effort; temperature1.0, top_p0.95, at least32768 output tokens. Validation recipe BF16, TP1, FlashAttention3 is a serving recommendation, not an assumed separate evaluation run. HLE tool mode, SWE scaffold and SciCode main/subproblem level are not specified; retained without inference. No reference-model, AA-LCR or throughput values imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Terminal-Bench 2.1 Agentic terminal use",
      "column_label": "K2-Horizon-7B",
      "row": [
        "Terminal-Bench 2.1 Agentic terminal use",
        "39.1",
        "Granite 4.2-8B 18.4",
        "Gemma 4-12B 27.3",
        "Qwen3.5-9B 29.2"
      ],
      "column_index": 1,
      "raw_value": "39.1",
      "scale_divisor": 100
    },
    "mode": "high"
  },
  {
    "model": "smollm3-3b",
    "bench": "mmlu",
    "score": 0.5984,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. 5-shot from subject dev sets, chat-formatted conversation; constrained A/B/C/D logits and tokenizer-robust extraction. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "MMLU",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "MMLU",
        "64.84",
        "64.42",
        "60.35",
        "59.84",
        "58.35",
        "72.25",
        "66.79"
      ],
      "raw_value": "59.84",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "mmlu-pro",
    "score": 0.239,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. 5-shot CoT validation examples in chat format; N/A choices removed; greedy decoding, max8192; custom final-letter parser. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "MMLU-Pro",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "MMLU-Pro",
        "37.42",
        "25.96",
        "22.25",
        "23.90",
        "34.76",
        "52.31",
        "32.03"
      ],
      "raw_value": "23.90",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "gpqa-diamond",
    "score": 0.2631,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. AppendixC explicitly identifies GPQA Diamond;10 answer-order permutations/question, chat format, constrained answer logits; mean permutation accuracy. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "GPQA",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "GPQA",
        "29.29",
        "26.57",
        "30.60",
        "26.31",
        "29.51",
        "34.85",
        "26.46"
      ],
      "raw_value": "26.31",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "ifeval-mean-four",
    "score": 0.7243999999999999,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. Official IFEval reference implementation; greedy max4096; arithmetic mean of strict/loose prompt/instruction accuracy. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "IFEval",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "IFEval",
        "77.58",
        "79.56",
        "71.43",
        "72.44",
        "76.85",
        "85.62",
        "81.06"
      ],
      "raw_value": "72.44",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "ifbench-unspecified",
    "score": 0.1793,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. allenai/IFBench_test official reference; greedy max4096. AppendixC mentions strict/loose prompt/instruction measures but does not explicitly define a final arithmetic aggregate; no strict-prompt claim. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "IFBench",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "IFBench",
        "25.85",
        "22.19",
        "20.78",
        "17.93",
        "23.53",
        "30.28",
        "18.37"
      ],
      "raw_value": "17.93",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "multi-if-three-turn-average",
    "score": 0.5886,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. facebook/Multi-IF official reference, truncate beyond model context; greedy max32768; average accuracy across3 turns. Strict/loose aggregation unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "Multi-IF",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "Multi-IF",
        "58.19",
        "60.26",
        "50.91",
        "58.86",
        "66.61",
        "75.54",
        "52.99"
      ],
      "raw_value": "58.86",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "gsm8k-loose-substring",
    "score": 0.8112,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. 5-shot random train examples, chat template, no system; greedy max4096; custom numeric parser then loose substring scoring. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "GSM8K",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "GSM8K",
        "84.38",
        "82.41",
        "75.21",
        "81.12",
        "89.92",
        "68.46",
        "82.64"
      ],
      "raw_value": "81.12",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "gsm-plus",
    "score": 0.5891,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. qintongli/GSM-Plus;5-shot, greedy max4096; GSM8K-style custom numeric parser; strict matching only. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "GSMPlus",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "GSMPlus",
        "64.76",
        "60.75",
        "38.68",
        "58.91",
        "68.38",
        "56.16",
        "59.14"
      ],
      "raw_value": "58.91",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "math-500",
    "score": 0.736,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. HuggingFaceH4/MATH-500;4 fixed shots and chat template, greedy max4096; custom numeric parser. MATH Level5 not also imported. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "MATH 500",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "MATH 500",
        "74.20",
        "63.60",
        "41.20",
        "73.60",
        "73.20",
        "85.60",
        "58.20"
      ],
      "raw_value": "73.60",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.5002,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. openai/MMMLU;0-shot without chat template; AR_XY/DE_DE/ES_LA/FR_FR/JA_JP/KO_KR/ZH_CN, constrained option logits; seven-language mean. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "MMMLU",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "MMMLU",
        "55.26",
        "55.39",
        "47.92",
        "50.02",
        "50.14",
        "60.67",
        "56.13"
      ],
      "raw_value": "50.02",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.6872,
    "source": "https://arxiv.org/html/2511.23404v1",
    "source_id": "report-2511.23404v1",
    "by": "independent",
    "evaluator": "Liquid AI",
    "date": "2026-09-25",
    "mode": "not-specified",
    "note": "Liquid AI own internal-harness evaluation of SmolLM3-3B chat checkpoint, Table7. Thinking toggle is not specified; no /think or /no_think attribution. juletxara/mgsm;5-shot per language; greedy max4096; six-language ES/FR/DE/ZH/JA/RU mean loose substring accuracy. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "row_label": "MGSM",
      "column_label": "SmolLM3-3B",
      "column_index": 4,
      "row": [
        "MGSM",
        "72.40",
        "74.32",
        "61.68",
        "68.72",
        "87.28",
        "81.76",
        "73.68"
      ],
      "raw_value": "68.72",
      "scale_divisor": 100,
      "methods_url": "https://arxiv.org/html/2511.23404v1#A3"
    }
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.3786,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "GPQA Diamond",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 1,
      "raw_value": "37.86 (± 0.83)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.83,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "mmlu-pro",
    "score": 0.4965,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "MMLU-Pro",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 2,
      "raw_value": "49.65 (± 0.18)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.18,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "ifeval-mean-four",
    "score": 0.8842,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "IFEval",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 3,
      "raw_value": "88.42 (± 0.35)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.35,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "ifbench-unspecified",
    "score": 0.4485,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "IFBench",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 4,
      "raw_value": "44.85 (± 0.73)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.73,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "gsm8k",
    "score": 0.856,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "GSM8K",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 6,
      "raw_value": "85.60 (± 0.00)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "math-500",
    "score": 0.8796,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "MATH-500",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 7,
      "raw_value": "87.96 (± 0.72)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.72,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "aime-2025",
    "score": 0.3173,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "AIME25",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 8,
      "raw_value": "31.73 (± 1.81)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 1.81,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "bfcl-v3",
    "score": 0.5697,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact LFM2.5-1.2B-Thinking checkpoint, own publisher-evaluated row. Mean of5 runs at temperature0.6; ±values are standard deviations, preserved in evidence. IFEval is mean of strict/loose prompt/instruction accuracy. BFCLv3 is the final weighted average using Liquid’s custom handler. GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology, but the values are Liquid’s own evaluations, not copied AA comparisons. Per-task token budgets/shots are not stated; none are inferred from the older LFM2 report. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "LFM2.5-1.2B-Thinking",
      "column_label": "BFCLv3",
      "row": [
        "LFM2.5-1.2B-Thinking",
        "37.86 (± 0.83)",
        "49.65 (± 0.18)",
        "88.42 (± 0.35)",
        "44.85 (± 0.73)",
        "69.33 (± 0.09)",
        "85.60 (± 0.00)",
        "87.96 (± 0.72)",
        "31.73 (± 1.81)",
        "56.97 (± 0.30)"
      ],
      "column_index": 9,
      "raw_value": "56.97 (± 0.30)",
      "scale_divisor": 100,
      "run_count": 5,
      "statistic": "mean",
      "spread": "standard deviation",
      "standard_deviation_percent": 0.3,
      "evaluation_temperature": 0.6
    },
    "mode": "thinking",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "ifeval-unspecified",
    "score": 0.9184,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "IFEval",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "IFEval",
        "79.44",
        "91.84",
        "+12.40"
      ],
      "column_index": 2,
      "raw_value": "91.84",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "ifbench-unspecified",
    "score": 0.5647,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "IFBench",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "IFBench",
        "26.00",
        "56.47",
        "+30.47"
      ],
      "column_index": 2,
      "raw_value": "56.47",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "math-500",
    "score": 0.8876,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "MATH500",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "MATH500",
        "74.80",
        "88.76",
        "+13.96"
      ],
      "column_index": 2,
      "raw_value": "88.76",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "aime-2025",
    "score": 0.4253,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "AIME25",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "AIME25",
        "20.00",
        "42.53",
        "+22.53"
      ],
      "column_index": 2,
      "raw_value": "42.53",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "tau2-telecom",
    "score": 0.8807,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "Tau² Telecom",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "Tau² Telecom",
        "13.60",
        "88.07",
        "+74.47"
      ],
      "column_index": 2,
      "raw_value": "88.07",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "tau2-retail",
    "score": 0.3982,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "row_label": "Tau² Retail",
      "column_label": "LFM2.5-8B-A1B",
      "row": [
        "Tau² Retail",
        "7.02",
        "39.82",
        "+32.80"
      ],
      "column_index": 2,
      "raw_value": "39.82",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "aime-2026",
    "score": 0.5,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "source_id": "card-liquidai-lfm2-5-8b-a1b",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Exact reasoning-tuned LFM2.5-8B-A1B release, own publisher row. Recommended generation temperature0.2/top_k80/repetition_penalty1.05 are not represented as exact benchmark run logs. The card does not state IFEval or IFBench aggregation; no method is borrowed from the1.2B card or older report. AA-Omniscience metrics and comparison-model rows excluded. BFCLv3/v4 withheld because two own-model tables disagree without explanation. τ² Retail/Telecom domains retained because no aggregate is supplied. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "row_label": "LFM2.5-8B-A1B",
      "column_label": "AIME26",
      "row": [
        "LFM2.5-8B-A1B",
        "8B/A1B",
        "88.76",
        "42.53",
        "50.00",
        "64.79",
        "49.73",
        "88.07",
        "39.82"
      ],
      "column_index": 4,
      "raw_value": "50.00",
      "scale_divisor": 100
    },
    "mode": "reasoning",
    "licence_condition_note": "LFM Open License v1.0: commercial reuse is subject to its Legal Entity and annual-revenue conditions, plus preservation of full licence, change and attribution notices. Each recipient must assess its own eligibility."
  },
  {
    "model": "addis-voice-2",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.044406280667320906,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "lab",
    "date": "2026-08-12",
    "note": "primary_no_numeric/addis: 362/8152 character edits. Model addis-voice-2; voice am-yohanes-calm; language condition supported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.044406280667320906,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "addis-voice-2",
      "voice": "am-yohanes-calm",
      "language": "am",
      "language_condition": "supported"
    },
    "counts": {
      "character_errors": 362,
      "reference_characters": 8152,
      "word_errors": 292,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.13865147198480532,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "azure-ameha-neural",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.030299313052011775,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/azure-ameha: 247/8152 character edits. Model am-ET-AmehaNeural; voice am-ET-AmehaNeural; language condition supported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.030299313052011775,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "am-ET-AmehaNeural",
      "voice": "am-ET-AmehaNeural",
      "language": "am",
      "language_condition": "supported"
    },
    "counts": {
      "character_errors": 247,
      "reference_characters": 8152,
      "word_errors": 235,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.11158594491927826,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "azure-mekdes-neural",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.041462217860647695,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/azure-mekdes: 338/8152 character edits. Model am-ET-MekdesNeural; voice am-ET-MekdesNeural; language condition supported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.041462217860647695,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "am-ET-MekdesNeural",
      "voice": "am-ET-MekdesNeural",
      "language": "am",
      "language_condition": "supported"
    },
    "counts": {
      "character_errors": 338,
      "reference_characters": 8152,
      "word_errors": 274,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.13010446343779677,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.028949950932286556,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/gemini: 236/8152 character edits. Model gemini-3.1-flash-tts-preview; voice Kore; language condition supported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.028949950932286556,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "gemini-3.1-flash-tts-preview",
      "voice": "Kore",
      "language": "am",
      "language_condition": "supported"
    },
    "counts": {
      "character_errors": 236,
      "reference_characters": 8152,
      "word_errors": 232,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.11016144349477683,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.1874386653581943,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/openai: 1528/8152 character edits. Model gpt-4o-mini-tts; voice coral; language condition unsupported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.1874386653581943,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "gpt-4o-mini-tts",
      "voice": "coral",
      "language": "am",
      "language_condition": "unsupported"
    },
    "counts": {
      "character_errors": 1528,
      "reference_characters": 8152,
      "word_errors": 660,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.31339031339031337,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "elevenlabs-v3",
    "bench": "amharic-addis-primary87-cer",
    "score": 1.1165358194308146,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/elevenlabs: 9102/8152 character edits. Model eleven_v3; voice 21m00Tcm4TlvDq8ikWAM; language condition unsupported. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 1.1165358194308146,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "eleven_v3",
      "voice": "21m00Tcm4TlvDq8ikWAM",
      "language": "am",
      "language_condition": "unsupported"
    },
    "counts": {
      "character_errors": 9102,
      "reference_characters": 8152,
      "word_errors": 3114,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 1.4786324786324787,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "mms-tts-amh",
    "bench": "amharic-addis-primary87-cer",
    "score": 0.07863101079489695,
    "source": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json",
    "source_id": "speech-amharic-addis-bysa",
    "by": "independent",
    "date": "2026-08-12",
    "note": "primary_no_numeric/meta-mms: 641/8152 character edits. Model facebook/mms-tts-amh; voice single-speaker; language condition open research model. Micro CER, not STT performance. Recomputed from all corresponding raw transcripts.",
    "raw_value": 0.07863101079489695,
    "raw_unit": "character error ratio",
    "sample_count": 87,
    "configuration": {
      "model": "facebook/mms-tts-amh",
      "voice": "single-speaker",
      "language": "am",
      "language_condition": "open research model"
    },
    "counts": {
      "character_errors": 641,
      "reference_characters": 8152,
      "word_errors": 482,
      "reference_words": 2106
    },
    "companion_metric": {
      "word_error_rate": 0.2288698955365622,
      "ranked": false
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "vibevoice-1-5b",
    "bench": "vibevoice-mit-figure-preference-mos",
    "score": 0.6075,
    "source": "https://github.com/microsoft/VibeVoice/blob/1541f590c7099820f10ea012f48d2399282df69f/Figures/VibeVoice-TTS-results.jpg",
    "source_id": "speech-vibevoice-mit-figure",
    "by": "lab",
    "date": "2026-09-25",
    "date_basis": "snapshot reviewed; legacy 2025 evaluation date not explicitly provided",
    "raw_value": 3.43,
    "raw_unit": "MOS on 1–5 preference scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "Exact printed Preference bar label 3.43 for VibeVoice-1.5B. MIT figure snapshot, not later arXiv table. Normalized (MOS-1)/4 for storage; no chance floor. Eight-transcript human listening study, not short-form Seed-TTS WER."
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "vibevoice-mit-figure-preference-mos",
    "score": 0.6625,
    "source": "https://github.com/microsoft/VibeVoice/blob/1541f590c7099820f10ea012f48d2399282df69f/Figures/VibeVoice-TTS-results.jpg",
    "source_id": "speech-vibevoice-mit-figure",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "snapshot reviewed; legacy 2025 evaluation date not explicitly provided",
    "raw_value": 3.65,
    "raw_unit": "MOS on 1–5 preference scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "Exact printed Preference bar label 3.65 for Gemini-2.5-Pro-Preview-TTS. MIT figure snapshot, not later arXiv table. Normalized (MOS-1)/4 for storage; no chance floor. Eight-transcript human listening study, not short-form Seed-TTS WER."
  },
  {
    "model": "elevenlabs-v3-alpha",
    "bench": "vibevoice-mit-figure-preference-mos",
    "score": 0.5925,
    "source": "https://github.com/microsoft/VibeVoice/blob/1541f590c7099820f10ea012f48d2399282df69f/Figures/VibeVoice-TTS-results.jpg",
    "source_id": "speech-vibevoice-mit-figure",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "snapshot reviewed; legacy 2025 evaluation date not explicitly provided",
    "raw_value": 3.37,
    "raw_unit": "MOS on 1–5 preference scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "Exact printed Preference bar label 3.37 for Eleven-v3 (Alpha). MIT figure snapshot, not later arXiv table. Normalized (MOS-1)/4 for storage; no chance floor. Eight-transcript human listening study, not short-form Seed-TTS WER."
  },
  {
    "model": "elevenlabs-v3",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1281.14,
    "raw_value": 1281.14,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "elevenlabs-eleven-v3",
    "source_model_label": "Eleven v3",
    "source_registry_request_id": "eleven_v3",
    "counts": {
      "wins": 582,
      "losses": 358,
      "ties": 103,
      "voteCount": 1043
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "speechify-simba-3-2",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1279.19,
    "raw_value": 1279.19,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID; frozen arena ID simba-3-2. Registry states vendor-rendered arena clips. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "speechify-simba-3-2",
    "source_model_label": "Simba 3.2",
    "source_registry_request_id": "simba-3.2",
    "counts": {
      "wins": 541,
      "losses": 347,
      "ties": 124,
      "voteCount": 1012
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "minimax-speech-2-8-turbo",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1269.46,
    "raw_value": 1269.46,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Vapi registry explicitly reports MiniMax confirmed the benchmarked generation was Speech 2.8, turbo tier (2026-07-23). This mapping relies on that attributed confirmation, not the older API request alias. Preserve request ID speech-2.5-turbo-preview; no claim of immutable backend identity. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "minimax-minimax-tts",
    "source_model_label": "Speech 2.8",
    "source_registry_request_id": "speech-2.5-turbo-preview",
    "counts": {
      "wins": 558,
      "losses": 368,
      "ties": 119,
      "voteCount": 1045
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "minimax-speech-02-hd",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1260.72,
    "raw_value": 1260.72,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "minimax-speech-02-hd",
    "source_model_label": "Speech 2 HD",
    "source_registry_request_id": "speech-02-hd",
    "counts": {
      "wins": 539,
      "losses": 378,
      "ties": 114,
      "voteCount": 1031
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1197.82,
    "raw_value": 1197.82,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Registry explicitly links Realtime TTS-2 launch; apiModelId/arenaApiId tts-2. Pipeline vendor ID override inworld-tts-2. Registry describes initial research preview; no immutable backend build supplied. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "inworld-tts-2",
    "source_model_label": "TTS-2",
    "source_registry_request_id": "tts-2",
    "counts": {
      "wins": 459,
      "losses": 485,
      "ties": 126,
      "voteCount": 1070
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1190.99,
    "raw_value": 1190.99,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "elevenlabs-flash-v25",
    "source_model_label": "Flash v2.5",
    "source_registry_request_id": "eleven_flash_v2_5",
    "counts": {
      "wins": 432,
      "losses": 486,
      "ties": 140,
      "voteCount": 1058
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "elevenlabs-turbo-v2-5",
    "bench": "vapi-humanness-export-20260610",
    "score": 1234.22,
    "raw_value": 1234.22,
    "raw_unit": "rating points",
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "source_id": "speech-vapi-humanness-20260610",
    "by": "independent",
    "date": "2026-06-10",
    "date_basis": "explicit production export date",
    "note": "Exact registry model ID. Retired from live sampling 2026-07-23, so only its actual June 10 production export is used. Historical production export; 98 battles, not a fabricated local seed.",
    "source_model_id": "elevenlabs-turbo-v25",
    "source_model_label": "Eleven Turbo v2.5",
    "source_registry_request_id": "eleven_turbo_v2_5",
    "counts": {
      "wins": 55,
      "losses": 38,
      "ties": 5,
      "voteCount": 98
    },
    "snapshot_total_unique_votes": 721,
    "data_license": "CC BY 4.0"
  },
  {
    "model": "granite-speech-4-1-2b",
    "bench": "granite41-lscln-punctuation-error",
    "score": 0.257,
    "raw_value": 25.7,
    "raw_unit": "percent punctuation error",
    "source": "https://huggingface.co/ibm-granite/granite-speech-4.1-2b/blob/de575db64086f84fdc79da4932d1076e965bc546/README.md",
    "source_id": "speech-granite41-punctuation-apache",
    "by": "lab",
    "date": "2026-09-25",
    "date_basis": "model-card snapshot review; exact evaluation date unspecified",
    "note": "Exact own-evaluation table row LScln, PER 25.70. Divide by 100. This is punctuation error, not WER. Base model, not plus or nar.",
    "companion_metric": {
      "capitalization_f1_percent": 89.71,
      "ranked": false
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "flux-1-schnell",
    "bench": "geneval",
    "score": 0.677,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#flux1-schnell-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:geneval",
    "evaluation_run_group": "visual-gap-amd-mxfp:flux-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "FLUX.1-schnell, BF16, 4 inference steps; AMD rerun on official GenEval 553-prompt dataset/protocol. Retain overall 0.6770, not GenEval accuracy 66.37%. Resolution, images per prompt, CFG, sampler, seeds and prompt rewriting are not separately disclosed. COCO2017 5,000 samples applies to CLIP/HPS columns only.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "FLUX.1-schnell (step-4), BF16",
      "table": "Table 2",
      "reported_value": "0.6770",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.3242",
        "0.2713",
        "66.37%",
        "0.6770"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "FLUX.1-schnell, BF16, 4 inference steps; AMD rerun on official GenEval 553-prompt dataset/protocol. Retain overall 0.6770, not GenEval accuracy 66.37%. Resolution, images per prompt, CFG, sampler, seeds and prompt rewriting are not separately disclosed. COCO2017 5,000 samples applies to CLIP/HPS columns only.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-image",
      "precision": "BF16",
      "steps": 4,
      "model_reference": "https://huggingface.co/black-forest-labs/FLUX.1-schnell",
      "generation_settings": {
        "resolution": null,
        "samples_per_prompt": null,
        "cfg": null,
        "seed": null,
        "prompt_rewrite": null
      }
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-subject-consistency-amd",
    "score": 0.9366,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:vbench-t2v-subject-consistency-amd",
    "evaluation_run_group": "visual-gap-amd-mxfp:wan21-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 subject consistency dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B, BF16",
      "table": "Table 3",
      "reported_value": "0.9366",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.9366",
        "0.9478",
        "0.9888",
        "0.6271",
        "0.6556",
        "0.8311"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 subject consistency dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-video",
      "precision": "BF16",
      "dimension": "subject-consistency",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
      "generation_settings": {
        "resolution": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_columns": {
        "Average Score": "0.8311 is not the official VBench total; dynamic degree is mentioned in prose but not tabulated."
      }
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-background-consistency-amd",
    "score": 0.9478,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:vbench-t2v-background-consistency-amd",
    "evaluation_run_group": "visual-gap-amd-mxfp:wan21-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 background consistency dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B, BF16",
      "table": "Table 3",
      "reported_value": "0.9478",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.9366",
        "0.9478",
        "0.9888",
        "0.6271",
        "0.6556",
        "0.8311"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 background consistency dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-video",
      "precision": "BF16",
      "dimension": "background-consistency",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
      "generation_settings": {
        "resolution": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_columns": {
        "Average Score": "0.8311 is not the official VBench total; dynamic degree is mentioned in prose but not tabulated."
      }
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-motion-smoothness-amd",
    "score": 0.9888,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:vbench-t2v-motion-smoothness-amd",
    "evaluation_run_group": "visual-gap-amd-mxfp:wan21-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 motion smoothness dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B, BF16",
      "table": "Table 3",
      "reported_value": "0.9888",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.9366",
        "0.9478",
        "0.9888",
        "0.6271",
        "0.6556",
        "0.8311"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 motion smoothness dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-video",
      "precision": "BF16",
      "dimension": "motion-smoothness",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
      "generation_settings": {
        "resolution": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_columns": {
        "Average Score": "0.8311 is not the official VBench total; dynamic degree is mentioned in prose but not tabulated."
      }
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-aesthetic-quality-amd",
    "score": 0.6271,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:vbench-t2v-aesthetic-quality-amd",
    "evaluation_run_group": "visual-gap-amd-mxfp:wan21-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 aesthetic quality dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B, BF16",
      "table": "Table 3",
      "reported_value": "0.6271",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.9366",
        "0.9478",
        "0.9888",
        "0.6271",
        "0.6556",
        "0.8311"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 aesthetic quality dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-video",
      "precision": "BF16",
      "dimension": "aesthetic-quality",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
      "generation_settings": {
        "resolution": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_columns": {
        "Average Score": "0.8311 is not the official VBench total; dynamic degree is mentioned in prose but not tabulated."
      }
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-imaging-quality-amd",
    "score": 0.6556,
    "source": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark",
    "source_id": "visual-gap-amd-mxfp",
    "source_group": "visual-gap-amd-mxfp:vbench-t2v-imaging-quality-amd",
    "evaluation_run_group": "visual-gap-amd-mxfp:wan21-bf16",
    "evaluator": "AMD ROCm blog authors",
    "by": "independent",
    "date": "2026-01-07",
    "note": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 imaging quality dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
    "data_license": "MIT",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B, BF16",
      "table": "Table 3",
      "reported_value": "0.6556",
      "divisor": 1,
      "source_row": [
        "BF16",
        "0.9366",
        "0.9478",
        "0.9888",
        "0.6271",
        "0.6556",
        "0.8311"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#how-about-the-quality",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.1-T2V-14B original BF16 baseline; AMD rerun, VBench 1.0 imaging quality dimension on multiple generated-video sets. Prompt list/count, frames, resolution, FPS, steps, CFG and seeds not disclosed for this measurement. This is a dimension, not the official total. Table’s 0.8311 Average Score is excluded; demonstration prompts/settings are not assumed to be the benchmark protocol.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "4263b2beebeec056f11720de17e8a58cb20cd5d7bc94f2f8c0cabc59834a4b94",
      "task": "text-to-video",
      "precision": "BF16",
      "dimension": "imaging-quality",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
      "generation_settings": {
        "resolution": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_columns": {
        "Average Score": "0.8311 is not the official VBench total; dynamic degree is mentioned in prose but not tabulated."
      }
    }
  },
  {
    "model": "hunyuanvideo",
    "bench": "vbench-t2v-total",
    "score": 0.8274,
    "source": "https://arxiv.org/pdf/2505.16864v2#page=8",
    "source_id": "visual-gap-jenga",
    "source_group": "visual-gap-jenga:vbench-t2v-total",
    "evaluation_run_group": "visual-gap-jenga:hunyuan-standard",
    "evaluator": "Jenga authors (CUHK / HKUST / Kuaishou / SmartMore)",
    "by": "independent",
    "date": "2025-11-22",
    "note": "Original HunyuanVideo T2V baseline rerun by Jenga authors; 50 NFE, 125 frames at 720×1280, original full-set VBench prompts, fixed seed per prompt (seed value undisclosed), one NVIDIA H800 unless otherwise specified. Table 5 identifies this as Standard prompts. FPS, CFG, precision and checkpoint revision not reported here. No Jenga/other acceleration, HunyuanI2V, 1.5, enhanced prompts or Tencent high-quality unreleased demo mapping.",
    "data_license": "CC-BY-SA-4.0",
    "evidence": {
      "source_model_label": "HunyuanVideo [12] (vanilla T2V baseline)",
      "table": "Table 1, PDF page 8; Table 5 standard prompts, PDF page 17",
      "reported_value": "82.74",
      "divisor": 100,
      "source_row": [
        "HunyuanVideo [12]",
        "50",
        "534.44",
        "10.68",
        "82.74%",
        "85.21%",
        "72.84%",
        "30.67",
        "1625s",
        "1.00×"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/pdf/2505.16864v2#page=8",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Original HunyuanVideo T2V baseline rerun by Jenga authors; 50 NFE, 125 frames at 720×1280, original full-set VBench prompts, fixed seed per prompt (seed value undisclosed), one NVIDIA H800 unless otherwise specified. Table 5 identifies this as Standard prompts. FPS, CFG, precision and checkpoint revision not reported here. No Jenga/other acceleration, HunyuanI2V, 1.5, enhanced prompts or Tencent high-quality unreleased demo mapping.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "81e5fc73bd50f88bba235bea55358e82e9b8e1ebf9e1d24710aa3b0c672d33e3",
      "task": "text-to-video",
      "steps": 50,
      "frames": 125,
      "height": 720,
      "width": 1280,
      "prompt_format": "standard/original",
      "model_reference": "https://github.com/Tencent-Hunyuan/HunyuanVideo",
      "generation_settings": {
        "fps": null,
        "cfg": null,
        "precision": null,
        "seed": "fixed per prompt; actual values unspecified"
      },
      "supporting_locations": [
        "https://arxiv.org/pdf/2505.16864v2#page=6",
        "https://arxiv.org/pdf/2505.16864v2#page=7",
        "https://arxiv.org/pdf/2505.16864v2#page=17"
      ]
    },
    "licensed_artifact": "/data/visual-gap-cc-by-sa.json"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "vbench-t2v-total-mobilewan-480p",
    "score": 0.813,
    "source": "https://arxiv.org/html/2607.06173v1#A1.T10",
    "source_id": "visual-gap-mobilewan",
    "source_group": "visual-gap-mobilewan:vbench-t2v-total-mobilewan-480p",
    "evaluation_run_group": "visual-gap-mobilewan:original-wan22-480p",
    "evaluator": "MobileWan authors / Qualcomm",
    "by": "independent",
    "date": "2026-07-07",
    "note": "Original released Wan2.2-TI2V-5B evaluated in T2V mode by MobileWan authors before fine-tuning, at non-native 480×832, 81 frames, 16 FPS (approximately 5 s). The authors explicitly report unsupported-480p/shaky-camera and reduced-motion limitations. Table 10 total 81.30; not native-resolution 83.28 or fine-tuned 83.12. Full prompt/sample protocol, inference steps, precision, CFG, sampler and seeds are not specified for this table.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Original Wan2.2 5B, 81×480×832",
      "table": "Table 10, Appendix A.4",
      "reported_value": "81.30",
      "divisor": 100,
      "source_row": [
        "Original Wan2.2 5B",
        "81×480×832",
        "8190",
        "82.46",
        "76.64",
        "81.30"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2607.06173v1#A1.SS4.p1",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Original released Wan2.2-TI2V-5B evaluated in T2V mode by MobileWan authors before fine-tuning, at non-native 480×832, 81 frames, 16 FPS (approximately 5 s). The authors explicitly report unsupported-480p/shaky-camera and reduced-motion limitations. Table 10 total 81.30; not native-resolution 83.28 or fine-tuned 83.12. Full prompt/sample protocol, inference steps, precision, CFG, sampler and seeds are not specified for this table.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "d6ab859805673efde31ffec20fd7db052ed4132ba27df5018b6cf0028f250a50",
      "task": "text-to-video",
      "frames": 81,
      "height": 480,
      "width": 832,
      "fps": 16,
      "native_resolution": false,
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B",
      "identity_evidence": "https://arxiv.org/html/2607.06173v1#A3",
      "generation_settings": {
        "steps": null,
        "precision": null,
        "cfg": null,
        "seed": null,
        "prompt_list": null,
        "sample_count": null
      },
      "excluded_rows": {
        "native_720p": "83.28 matches an earlier published comparison; not needed and per-row original rerun not separately verified.",
        "finetuned_480p": "83.12 belongs to fine-tuned checkpoint, not the requested original model."
      }
    }
  },
  {
    "model": "wan-2-2-5b",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.084,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan22-base-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.2-TI2V-5B original base checkpoint; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text. Table 3 reports 8 s / 720p and VGGT world decoder; published WO 0.084. Section 4.3 links the base checkpoint to TI2V-5B and distinguishes the authors’ FT/FT-DA variants. Inference frame count, exact pixel dimensions, FPS, CFG, steps and seed are not stated for this row; 121 frames in Section 4.3 describes training sampling and is not assumed for inference. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.2-5B-base (Wan2.2-TI2V-5B)",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.084",
      "divisor": 1,
      "source_row": [
        "Wan2.2-5B-base",
        "✓",
        "8 - 720p",
        "3.666",
        "1.636",
        "58.11",
        "0.012",
        "0.944",
        "0.084"
      ],
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.17792v2#S4.SS3.p1",
      "evaluator_relationship": "evaluator is not the evaluated checkpoint’s model author",
      "protocol": "Wan2.2-TI2V-5B original base checkpoint; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text. Table 3 reports 8 s / 720p and VGGT world decoder; published WO 0.084. Section 4.3 links the base checkpoint to TI2V-5B and distinguishes the authors’ FT/FT-DA variants. Inference frame count, exact pixel dimensions, FPS, CFG, steps and seed are not stated for this row; 121 frames in Section 4.3 describes training sampling and is not assumed for inference. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash. No hosted API equivalence or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video",
      "model_reference": "https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B",
      "identity_evidence": [
        "https://arxiv.org/html/2511.17792v2#S4.SS3.p1",
        "https://arxiv.org/html/2511.17792v2#S4.T5"
      ],
      "generation_settings": {
        "duration_seconds": 8,
        "resolution_label": "720p",
        "height": null,
        "width": null,
        "frames": null,
        "fps": null,
        "steps": null,
        "cfg": null,
        "seed": null
      },
      "benchmark_settings": {
        "test_scenarios": 125,
        "dataset_scenarios": 450,
        "semantic_categories": 47,
        "decoder": "VGGT",
        "fine_tuned": false,
        "task": "image-to-video planning"
      }
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "hellaswag",
    "score": 0.377,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; 0-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "HellaSwag",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "HellaSwag",
        "0-shot",
        "37.7"
      ],
      "raw_value": "37.7",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[HellaSwag][hellaswag]",
        "0-shot",
        "37.7"
      ]
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "piqa",
    "score": 0.662,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; 0-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "PIQA",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "PIQA",
        "0-shot",
        "66.2"
      ],
      "raw_value": "66.2",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[PIQA][piqa]",
        "0-shot",
        "66.2"
      ]
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "arc-challenge",
    "score": 0.282,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; 0-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "ARC-c",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "ARC-c",
        "0-shot",
        "28.2"
      ],
      "raw_value": "28.2",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[ARC-c][arc]",
        "0-shot",
        "28.2"
      ]
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "winogrande",
    "score": 0.523,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; 0-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "WinoGrande",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "WinoGrande",
        "0-shot",
        "52.3"
      ],
      "raw_value": "52.3",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[WinoGrande][winogrande]",
        "0-shot",
        "52.3"
      ]
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "bbh",
    "score": 0.267,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; few-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "BIG-Bench Hard",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "BIG-Bench Hard",
        "few-shot",
        "26.7"
      ],
      "raw_value": "26.7",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[BIG-Bench Hard][bbh]",
        "few-shot",
        "26.7"
      ]
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "ifeval-unspecified",
    "score": 0.512,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Gemma 3 IT 270M column; 0-shot. Exact IT checkpoint, not PT and not separately measured QAT. BBH few-shot count and IFEval aggregation are unspecified; decoding/scoring details not inferred. Existing model ID has IT local variants. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 9,
      "row_label": "IF Eval",
      "column_index": 2,
      "column_label": "Gemma 3 IT 270M",
      "row": [
        "IF Eval",
        "0-shot",
        "51.2"
      ],
      "raw_value": "51.2",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3-270m-it/blob/ac82b4e820549b854eebf28ce6dedaf9fdfa17b3/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[IF Eval][ifeval]",
        "0-shot",
        "51.2"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "mgsm",
    "score": 0.531,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly Accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "MGSM",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "MGSM",
        "Accuracy",
        "0-shot",
        "53.1",
        "60.7"
      ],
      "raw_value": "53.1",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[MGSM][mgsm]",
        "Accuracy",
        "0-shot",
        "53.1",
        "60.7"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "global-mmlu",
    "score": 0.551,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly Accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "row_label": "Global-MMLU",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "Global-MMLU",
        "Accuracy",
        "0-shot",
        "55.1",
        "60.3"
      ],
      "raw_value": "55.1",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 1,
      "mirror_row": [
        "[Global-MMLU][global-mmlu]",
        "Accuracy",
        "0-shot",
        "55.1",
        "60.3"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "gpqa-diamond-google-relaxed",
    "score": 0.248,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly RelaxedAccuracy/accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "GPQA Diamond",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "GPQA Diamond",
        "RelaxedAccuracy/accuracy",
        "0-shot",
        "24.8",
        "23.7"
      ],
      "raw_value": "24.8",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 2,
      "mirror_row": [
        "[GPQA][gpqa] Diamond",
        "RelaxedAccuracy/accuracy",
        "0-shot",
        "24.8",
        "23.7"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.18600000000000003,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly pass@1. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "LiveCodeBench v5",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "LiveCodeBench v5",
        "pass@1",
        "0-shot",
        "18.6",
        "25.7"
      ],
      "raw_value": "18.6",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 2,
      "mirror_row": [
        "[LiveCodeBench][lcb] v5",
        "pass@1",
        "0-shot",
        "18.6",
        "25.7"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "codegolf-v2-2",
    "score": 0.11,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly pass@1. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "Codegolf v2.2",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "Codegolf v2.2",
        "pass@1",
        "0-shot",
        "11.0",
        "16.8"
      ],
      "raw_value": "11.0",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 2,
      "mirror_row": [
        "Codegolf v2.2",
        "pass@1",
        "0-shot",
        "11.0",
        "16.8"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "aime-2025",
    "score": 0.067,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly Accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "row_label": "AIME 2025",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "AIME 2025",
        "Accuracy",
        "0-shot",
        "6.7",
        "11.6"
      ],
      "raw_value": "6.7",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 2,
      "mirror_row": [
        "[AIME 2025][aime-2025]",
        "Accuracy",
        "0-shot",
        "6.7",
        "11.6"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "mmlu",
    "score": 0.601,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly Accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "MMLU",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "MMLU",
        "Accuracy",
        "0-shot",
        "60.1",
        "64.9"
      ],
      "raw_value": "60.1",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 3,
      "mirror_row": [
        "[MMLU][mmlu]",
        "Accuracy",
        "0-shot",
        "60.1",
        "64.9"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "mbpp",
    "score": 0.5660000000000001,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 3-shot, metric exactly pass@1. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "MBPP",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "MBPP",
        "pass@1",
        "3-shot",
        "56.6",
        "63.6"
      ],
      "raw_value": "56.6",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 3,
      "mirror_row": [
        "[MBPP][mbpp]",
        "pass@1",
        "3-shot",
        "56.6",
        "63.6"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "humaneval",
    "score": 0.665,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly pass@1. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "HumanEval",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "HumanEval",
        "pass@1",
        "0-shot",
        "66.5",
        "75.0"
      ],
      "raw_value": "66.5",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 3,
      "mirror_row": [
        "[HumanEval][humaneval]",
        "pass@1",
        "0-shot",
        "66.5",
        "75.0"
      ]
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "mmlu-pro",
    "score": 0.405,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "E2B IT column; 0-shot, metric exactly Accuracy. No PT, E4B, separately quantized checkpoint, unversioned LCB, or overlapping Global-MMLU-Lite row imported. Decoding and language aggregation unspecified where absent from card. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "row_label": "MMLU (Pro)",
      "column_index": 3,
      "column_label": "E2B IT",
      "row": [
        "MMLU (Pro)",
        "Accuracy",
        "0-shot",
        "40.5",
        "50.6"
      ],
      "raw_value": "40.5",
      "scale_divisor": 100,
      "pinned_mirror_url": "https://huggingface.co/google/gemma-3n-E2B-it/blob/5e092ebca197cdcd8d8b195040accf22693501bc/README.md",
      "pinned_mirror_table_index": 3,
      "mirror_row": [
        "[MMLU][mmlu] (Pro)",
        "Accuracy",
        "0-shot",
        "40.5",
        "50.6"
      ]
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "mmlu",
    "score": 0.865,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "MMLU",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "86.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "mmlu-pro",
    "score": 0.784,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "MMLU-Pro",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "c-eval",
    "score": 0.9059999999999999,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "C-Eval",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "90.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "cmmlu",
    "score": 0.902,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "CMMLU",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "90.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "chinese-simpleqa",
    "score": 0.7709999999999999,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "ChineseSimpleQA",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "77.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "simpleqa",
    "score": 0.45899999999999996,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "SimpleQA",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "45.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "ifeval-unspecified",
    "score": 0.88,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "IFEval",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "88.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "multi-if-unspecified",
    "score": 0.7659999999999999,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "Multi-IF",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "76.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "sysbench-unspecified",
    "score": 0.698,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "Sysbench",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "69.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "math-500",
    "score": 0.9640000000000001,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "MATH-500",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "96.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "gsm8k",
    "score": 0.966,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "GSM8K",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "96.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "aime-2024",
    "score": 0.5479999999999999,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "AIME'24",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "54.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "aime-2025",
    "score": 0.35100000000000003,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "AIME'25",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "35.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "bbh",
    "score": 0.943,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "BBH",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "94.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "musr",
    "score": 0.6990000000000001,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "MUSR",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "69.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "livecodebench-unversioned",
    "score": 0.38799999999999996,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "LiveCodeBench",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "38.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "humaneval-plus",
    "score": 0.9209999999999999,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "Humaneval+",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "92.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-4-5-300b-a47b",
    "bench": "mbpp-plus",
    "score": 0.81,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "source_id": "card-ernie-45-posttrained-final",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "post-trained",
    "note": "Own rightmost ERNIE-4.5-300B-A47B column, explicitly post-trained README section. -PT denotes PyTorch, not pretraining; card prose and GitHub variant matrix resolve the HF configuration table’s contradictory Pretraining label. Generation settings and shot counts are not stated for this chart; recommendations are not treated as evaluation logs. Only expanded HumanEval+/MBPP+ scores imported. LCB window and IF aggregations remain unspecified. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "7bb443fbaf8e83f32c7f3920a87639a1a77ace3411048fedd3703def19377511",
      "row_label": "MBPP+",
      "column_label": "ERNIE-4.5-300B-A47B",
      "section": "Performance of post-trained model ERNIE-4.5-300B-A47B",
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.5660000000000001,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. SWE uses OpenHands with tailored prompts, temperature0.6,top_p0.95,max_new_tokens32K,context256K; SWEPro split unspecified. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "SWE-Bench Pro",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "56.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "swe-bench-multilingual",
    "score": 0.7240000000000001,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. SWE uses OpenHands with tailored prompts, temperature0.6,top_p0.95,max_new_tokens32K,context256K; SWEPro split unspecified. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "SWE-Bench Multilingual",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "72.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "mcp-atlas-v1-500",
    "score": 0.655,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. 500 public tasks; official v1 harness,20-turn limit; Gemini2.5Pro claim-coverage judge. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "MCP-Atlas",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "65.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "skillsbench-87",
    "score": 0.44799999999999995,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Kilo-code harness;87 tasks excluding external-API-dependent tasks; mean over3runs. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "SkillsBench",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "44.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "bfcl-v4",
    "score": 0.73,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated.  HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "BFCL-v4",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "widesearch-corrected",
    "score": 0.736,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Corrected WideSearch with official prompt; GPT4.1 judge; own internal ReAct search harness, average pass@1. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "WideSearch",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "73.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "browsecomp-ling-single-context",
    "score": 0.722,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Internal single-agent ReAct harness; summarize history at64K and resume; average pass@1. Separate from multi-agent82.0 score. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "BrowseComp (w/ ctx)",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "72.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "ifbench-unspecified",
    "score": 0.745,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Published IFBench score where card/report does not explicitly identify scoring aggregation. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "IFBench",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "74.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "sysbench-unspecified",
    "score": 0.9359999999999999,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Published SysBench system-instruction benchmark score; exact aggregation/snapshot unspecified. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "SysBench",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "93.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "aime-2026",
    "score": 0.932,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated.  HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "AIME26",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "93.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "hmmt-feb-2026",
    "score": 0.87,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. February 2026 HMMT math contest. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "HMMT-Feb26",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "87.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "imo-answerbench",
    "score": 0.8370000000000001,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. IMO-AnswerBench answer accuracy; no majority-vote aggregation assumed. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "IMO-AnswerBench",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "83.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "hle",
    "score": 0.22699999999999998,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated.  HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "HLE",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "22.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.828,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Source explicitly labels 2408–2505 window but not release version. Kept separate from a v6 claim. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "LiveCodeBench (2408-2505)",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ling-3-0-flash",
    "bench": "multi-if-unspecified",
    "score": 0.877,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "source_id": "card-final-ling-3-0-flash",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Own Ling-3.0-flash first numeric column, not Flash-base, Fin, VL or DSpark. Thinking enabled; default temperature0.6,top_p0.95,top_k20 unless task override stated. Published Multi-IF score; source does not resolve turn/strictness aggregation. HLE tools/text-image subset and IF aggregations not specified. AA-labelled tasks, GDPval Elo, Terminal AA protocol and MRCR length components omitted. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "asset_sha256": "a7dc4f02ecdfdeb6b2a2179b6b719f51655372b2b42810d98d245f6bf4999817",
      "row_label": "Multi-IF",
      "column_label": "Ling-3.0-flash",
      "column_index": 1,
      "raw_value": "87.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "tau2-retail",
    "score": 0.6667000000000001,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Apriel-1.6-15B-Thinker own internal evaluation, outside AA Index v3. All evaluations begin with Here are my reasoning steps:\\n per card. Temperature0.6 is a recommendation, not an asserted per-task log. Token limit, task scaffold/simulator versions and Multi-IF aggregation unspecified. The card’s copied Apriel1.5 citation is not used to relabel this1.6 result. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "column_index": 2,
      "column_label": "Apriel-1.6-15B-Thinker",
      "row": [
        "Function Calling",
        "Tau2 bench Retail",
        "66.67",
        "46.78",
        "61.4",
        "59.94",
        "61.69",
        "73.39",
        "69.8",
        "75.73"
      ],
      "raw_value": "66.67",
      "scale_divisor": 100
    }
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Apriel-1.6-15B-Thinker own internal evaluation, outside AA Index v3. All evaluations begin with Here are my reasoning steps:\\n per card. Temperature0.6 is a recommendation, not an asserted per-task log. Token limit, task scaffold/simulator versions and Multi-IF aggregation unspecified. The card’s copied Apriel1.5 citation is not used to relabel this1.6 result. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "column_index": 2,
      "column_label": "Apriel-1.6-15B-Thinker",
      "row": [
        "Function Calling",
        "Tau2 bench Airline",
        "58",
        "52",
        "45.3",
        "47.33",
        "56.66",
        "59.33",
        "58",
        "61.33"
      ],
      "raw_value": "58",
      "scale_divisor": 100
    }
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "multi-if-unspecified",
    "score": 0.8334,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Apriel-1.6-15B-Thinker own internal evaluation, outside AA Index v3. All evaluations begin with Here are my reasoning steps:\\n per card. Temperature0.6 is a recommendation, not an asserted per-task log. Token limit, task scaffold/simulator versions and Multi-IF aggregation unspecified. The card’s copied Apriel1.5 citation is not used to relabel this1.6 result. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "column_index": 2,
      "column_label": "Apriel-1.6-15B-Thinker",
      "row": [
        "Instruction Following",
        "Multi IF",
        "83.34",
        "76.91",
        "82.95",
        "73.76",
        "82.49",
        "85.37",
        "84.32",
        "87.28"
      ],
      "raw_value": "83.34",
      "scale_divisor": 100
    }
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "swe-bench-verified",
    "score": 0.23,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Apriel-1.6-15B-Thinker own internal evaluation, outside AA Index v3. All evaluations begin with Here are my reasoning steps:\\n per card. Temperature0.6 is a recommendation, not an asserted per-task log. Token limit, task scaffold/simulator versions and Multi-IF aggregation unspecified. The card’s copied Apriel1.5 citation is not used to relabel this1.6 result. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "column_index": 2,
      "column_label": "Apriel-1.6-15B-Thinker",
      "row": [
        "Agentic",
        "SWE Bench Verified",
        "23",
        "16",
        "31",
        "29.60",
        "34.20",
        "61",
        "64.2",
        "22.60"
      ],
      "raw_value": "23",
      "scale_divisor": 100
    }
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "aider-polyglot",
    "score": 0.3768,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Apriel-1.6-15B-Thinker own internal evaluation, outside AA Index v3. All evaluations begin with Here are my reasoning steps:\\n per card. Temperature0.6 is a recommendation, not an asserted per-task log. Token limit, task scaffold/simulator versions and Multi-IF aggregation unspecified. The card’s copied Apriel1.5 citation is not used to relabel this1.6 result. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "column_index": 2,
      "column_label": "Apriel-1.6-15B-Thinker",
      "row": [
        "Agentic",
        "Aider Polyglot",
        "37.68",
        "26.37",
        "42",
        "71.40",
        "40",
        "71.60",
        "78",
        "60.40"
      ],
      "raw_value": "37.68",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "mmlu",
    "score": 0.7490000000000001,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MMLU (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MMLU (Acc) {}_{\\text{(Acc)}}",
        "62.4",
        "62.1",
        "69.8",
        "72.1",
        "50.8",
        "73.2",
        "80.8",
        "79.8",
        "76.2",
        "76.7",
        "74.9",
        "83.4"
      ],
      "raw_value": "74.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "mmlu-pro",
    "score": 0.5760000000000001,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MMLU-Pro (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MMLU-Pro (Acc) {}_{\\text{(Acc)}}",
        "35.2",
        "38.3",
        "49.5",
        "49.5",
        "25.3",
        "54.5",
        "66.9",
        "63.9",
        "55.8",
        "57.1",
        "57.6",
        "70.7"
      ],
      "raw_value": "57.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "agieval-marco-unspecified",
    "score": 0.49,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "AGIEval (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "AGIEval (Acc) {}_{\\text{(Acc)}}",
        "39.6",
        "33.0",
        "44.7",
        "45.2",
        "30.7",
        "49.8",
        "51.7",
        "52.4",
        "43.6",
        "44.7",
        "49.0",
        "55.4"
      ],
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "gpqa-diamond",
    "score": 0.397,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "GPQA-Diamond (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "GPQA-Diamond (Acc) {}_{\\text{(Acc)}}",
        "27.5",
        "21.0",
        "31.6",
        "31.9",
        "28.3",
        "22.2",
        "50.8",
        "44.8",
        "35.2",
        "38.6",
        "39.7",
        "50.3"
      ],
      "raw_value": "39.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "gsm8k",
    "score": 0.872,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "GSM8K (EM) {}_{\\text{(EM)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "GSM8K (EM) {}_{\\text{(EM)}}",
        "77.9",
        "79.7",
        "79.0",
        "84.6",
        "71.1",
        "86.7",
        "88.6",
        "89.5",
        "89.7",
        "83.9",
        "87.2",
        "93.1"
      ],
      "raw_value": "87.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "math-subset-unspecified",
    "score": 0.8390000000000001,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MATH (EM) {}_{\\text{(EM)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MATH (EM) {}_{\\text{(EM)}}",
        "70.6",
        "73.7",
        "70.2",
        "82.6",
        "53.4",
        "79.6",
        "93.4",
        "86.2",
        "83.8",
        "75.7",
        "83.9",
        "91.8"
      ],
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "global-mmlu-marco",
    "score": 0.57,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Acc@1, one response/query. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "GlobalMMLU (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "GlobalMMLU (Acc) {}_{\\text{(Acc)}}",
        "46.3",
        "45.9",
        "38.4",
        "49.0",
        "43.0",
        "58.7",
        "70.2",
        "55.4",
        "69.2",
        "67.4",
        "57.0",
        "73.3"
      ],
      "raw_value": "57.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "mmmlu-marco",
    "score": 0.623,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Acc@1, one response/query. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MMMLU (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MMMLU (Acc) {}_{\\text{(Acc)}}",
        "49.0",
        "49.0",
        "39.4",
        "56.5",
        "44.1",
        "59.9",
        "71.3",
        "56.4",
        "69.4",
        "68.1",
        "62.3",
        "73.7"
      ],
      "raw_value": "62.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.433,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MMLU-ProX-Lite (Acc) {}_{\\text{(Acc)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MMLU-ProX-Lite (Acc) {}_{\\text{(Acc)}}",
        "28.6",
        "30.3",
        "26.7",
        "33.8",
        "22.1",
        "43.2",
        "58.3",
        "43.3",
        "51.3",
        "51.6",
        "43.3",
        "61.2"
      ],
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "mgsm-marco",
    "score": 0.6779999999999999,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "MGSM (EM) {}_{\\text{(EM)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "MGSM (EM) {}_{\\text{(EM)}}",
        "63.6",
        "67.6",
        "47.0",
        "56.5",
        "55.3",
        "76.5",
        "84.4",
        "68.7",
        "84.0",
        "75.7",
        "67.8",
        "87.4"
      ],
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "lfm2-24b-a2b",
    "bench": "polymath-marco",
    "score": 0.293,
    "source": "https://arxiv.org/html/2604.25578v1",
    "source_id": "report-marco-moe-260425578v1",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Table11 own Light-Eval baseline run. Exact LFM2-24B-A2B identified in Section4.3.1. Temperature1.0,top_p1.0,max8192 tokens. Avg@8 mean across8 responses/query, not pass@8 or majority voting. Shot count unspecified; MATH subset not resolved. No MMLU-Redux alongside near-overlapping MMLU; no table averages or language components added. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T11",
      "row_label": "PolyMath (EM) {}_{\\text{(EM)}}",
      "column_label": "LFM2 /24B-A2B /Instruct",
      "column_index": 11,
      "row": [
        "PolyMath (EM) {}_{\\text{(EM)}}",
        "23.4",
        "25.5",
        "16.3",
        "26.5",
        "18.7",
        "29.6",
        "47.2",
        "26.4",
        "35.5",
        "28.9",
        "29.3",
        "44.7"
      ],
      "raw_value": "29.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "thunder-konubench-symbol-zero-shot",
    "score": 0.265,
    "source": "https://aclanthology.org/2026.findings-acl.324.pdf",
    "source_id": "acl-konubench-2026",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "logit-scoring",
    "note": "Table16 (publication p6511, PDFpage21), zero-shot symbol accuracy using LM Evaluation Harness, BF16 (AppendixF). Scores are original checkpoint runs before benchmark fine-tuning; thinking toggle/chat-template details not specified for logit-based evaluation. Only zero-shot symbol condition selected. Cloze figures conflict between Table16 and Table23 and are excluded; selected symbol values agree after rounding. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "pdf-table",
      "table_number": 16,
      "pdf_page": 21,
      "printed_page": 6511,
      "row_label": "EXAONE-4.0-1.2B",
      "column_label": "0shot / symbol",
      "raw_line": "EXAONE-4.0-1.2B              32.2        26.5    44.1        29.2    49.1        41.6    60.2        43.7    65.8   44.3",
      "raw_value": "26.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "thunder-konubench-symbol-zero-shot",
    "score": 0.815,
    "source": "https://aclanthology.org/2026.findings-acl.324.pdf",
    "source_id": "acl-konubench-2026",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "logit-scoring",
    "note": "Table16 (publication p6511, PDFpage21), zero-shot symbol accuracy using LM Evaluation Harness, BF16 (AppendixF). Scores are original checkpoint runs before benchmark fine-tuning; thinking toggle/chat-template details not specified for logit-based evaluation. Only zero-shot symbol condition selected. Cloze figures conflict between Table16 and Table23 and are excluded; selected symbol values agree after rounding. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "pdf-table",
      "table_number": 16,
      "pdf_page": 21,
      "printed_page": 6511,
      "row_label": "EXAONE-4.0-32B",
      "column_label": "0shot / symbol",
      "raw_line": "EXAONE-4.0-32B               51.9        81.5    65.6        86.9    72.0        88.0    80.4        91.0    83.1   92.1",
      "raw_value": "81.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.364,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "source_id": "acl-kcl-2026",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "reasoning",
    "note": "Own Exaone4.0 32B Small Reasoning row, vanilla setting without supporting precedents. Mean of3runs; printed ±1.6 percentage points (dispersion definition not asserted). vLLM for this open-weight model; explicit reasoning category, exact reasoning budget/decoding unspecified. MCQA accuracy and rubric-weighted essay score kept separate; no precedents variant or group average duplicated. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "pdf-table",
      "table_number": 2,
      "pdf_page": 4,
      "printed_page": 235,
      "row_label": "Exaone 4.0 32B",
      "column_label": "vanilla",
      "reported_uncertainty_percentage_points": "1.6",
      "raw_value": "36.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "kcl-essay-vanilla",
    "score": 0.326,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "source_id": "acl-kcl-2026",
    "by": "independent",
    "date": "2026-09-25",
    "mode": "reasoning",
    "note": "Own Exaone4.0 32B Small Reasoning row, vanilla setting without supporting precedents. Mean of3runs; printed ±0.4 percentage points (dispersion definition not asserted). vLLM for this open-weight model; explicit reasoning category, exact reasoning budget/decoding unspecified. MCQA accuracy and rubric-weighted essay score kept separate; no precedents variant or group average duplicated. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "pdf-table",
      "table_number": 3,
      "pdf_page": 4,
      "printed_page": 235,
      "row_label": "Exaone 4.0 32B",
      "column_label": "vanilla",
      "reported_uncertainty_percentage_points": "0.4",
      "raw_value": "32.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.049489,
    "raw_value": 4.948889,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row MiniMax-Speech-2.8-HD, evaluated by the Qwen-Audio authors through MiniMax's public API (paper footnote platform.minimax.io models-intro); voice cloning from the CV3 prompt.",
    "components": {
      "zh": 3.42,
      "en": 3.45,
      "ja": 6.29,
      "ko": 7.49,
      "de": 3.3,
      "es": 2.79,
      "fr": 8.74,
      "it": 3.67,
      "ru": 5.39
    }
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-ext7-macro-err",
    "score": 0.027771,
    "raw_value": 2.777143,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4, the paper's seven-language extension of CV3-Eval (ar,id,pt,th,vi,ms,tl), WER. Derived unweighted mean; only systems with all seven languages included. Row MiniMax-Speech-2.8-HD, evaluated by the Qwen-Audio authors through MiniMax's public API (paper footnote platform.minimax.io models-intro); voice cloning from the CV3 prompt.",
    "components": {
      "ar": 3.38,
      "id": 1.46,
      "pt": 1.86,
      "th": 1.65,
      "vi": 1.64,
      "ms": 3.1,
      "tl": 6.35
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0513,
    "raw_value": 5.13,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row ElevenLabs-v3, evaluated by the Qwen-Audio authors through ElevenLabs' public API (footnote elevenlabs.io/docs/overview/models); voice cloning from the CV3 prompt.",
    "components": {
      "zh": 4.46,
      "en": 3.61,
      "ja": 5.71,
      "ko": 5.46,
      "de": 3.64,
      "es": 3.92,
      "fr": 9.31,
      "it": 4.68,
      "ru": 5.38
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-ext7-macro-err",
    "score": 0.0506,
    "raw_value": 5.06,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4, the paper's seven-language extension of CV3-Eval (ar,id,pt,th,vi,ms,tl), WER. Derived unweighted mean; only systems with all seven languages included. Row ElevenLabs-v3, evaluated by the Qwen-Audio authors through ElevenLabs' public API (footnote elevenlabs.io/docs/overview/models); voice cloning from the CV3 prompt.",
    "components": {
      "ar": 5.51,
      "id": 2.85,
      "pt": 2.33,
      "th": 3.44,
      "vi": 4.08,
      "ms": 4.41,
      "tl": 12.8
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.050967,
    "raw_value": 5.096667,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row Qwen3-TTS-12Hz-1.7B-Base, publicly released checkpoint run by the authors. Marked lab: authors (Alibaba Token Foundry) and Qwen3-TTS maker (Alibaba Qwen) share a parent company.",
    "components": {
      "zh": 3.09,
      "en": 3.67,
      "ja": 6.48,
      "ko": 5.64,
      "de": 3.31,
      "es": 3.22,
      "fr": 9.05,
      "it": 4.01,
      "ru": 7.4
    }
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.106289,
    "raw_value": 10.628889,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row Dots.TTS-2B (SOAR), publicly released checkpoint run by the Qwen-Audio authors.",
    "components": {
      "zh": 3.58,
      "en": 4.7,
      "ja": 8.23,
      "ko": 10.1,
      "de": 5.97,
      "es": 8.04,
      "fr": 35.7,
      "it": 5.34,
      "ru": 14
    }
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.062144,
    "raw_value": 6.214444,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row VoxCPM2, publicly released checkpoint run by the Qwen-Audio authors.",
    "components": {
      "zh": 3.55,
      "en": 6.21,
      "ja": 5.88,
      "ko": 9.95,
      "de": 5.48,
      "es": 4.17,
      "fr": 10.3,
      "it": 4.42,
      "ru": 5.97
    }
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-ext7-macro-err",
    "score": 0.042214,
    "raw_value": 4.221429,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4, the paper's seven-language extension of CV3-Eval (ar,id,pt,th,vi,ms,tl), WER. Derived unweighted mean; only systems with all seven languages included. Row VoxCPM2, publicly released checkpoint run by the Qwen-Audio authors.",
    "components": {
      "ar": 4.44,
      "id": 3.1,
      "pt": 2.63,
      "th": 1.86,
      "vi": 4.97,
      "ms": 5.47,
      "tl": 7.08
    }
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0467,
    "raw_value": 4.67,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used.",
    "components": {
      "zh": 3.35,
      "en": 4.25,
      "ja": 4.78,
      "ko": 4.3,
      "de": 4,
      "es": 3.08,
      "fr": 9.77,
      "it": 3.82,
      "ru": 4.68
    }
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-ext7-macro-err",
    "score": 0.030529,
    "raw_value": 3.052857,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4, the paper's seven-language extension of CV3-Eval (ar,id,pt,th,vi,ms,tl), WER. Derived unweighted mean; only systems with all seven languages included. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used.",
    "components": {
      "ar": 3.36,
      "id": 2.35,
      "pt": 1.99,
      "th": 1.45,
      "vi": 3.17,
      "ms": 2.62,
      "tl": 6.43
    }
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0742,
    "raw_value": 7.42,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 74.6% and DNSMOS 3.74 retained as companion metadata only. Row MiniMax-Speech-2.8-HD, evaluated by the Qwen-Audio authors through MiniMax's public API (paper footnote platform.minimax.io models-intro); voice cloning from the CV3 prompt.",
    "companion": {
      "sim_eres2net_percent": 74.6,
      "dnsmos": 3.74
    }
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0737,
    "raw_value": 7.37,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 72.1% and DNSMOS 3.80 retained as companion metadata only. Row MiniMax-Speech-2.8-HD, evaluated by the Qwen-Audio authors through MiniMax's public API (paper footnote platform.minimax.io models-intro); voice cloning from the CV3 prompt.",
    "companion": {
      "sim_eres2net_percent": 72.1,
      "dnsmos": 3.8
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.1066,
    "raw_value": 10.66,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 50.3% and DNSMOS 3.81 retained as companion metadata only. Row ElevenLabs-v3, evaluated by the Qwen-Audio authors through ElevenLabs' public API (footnote elevenlabs.io/docs/overview/models); voice cloning from the CV3 prompt.",
    "companion": {
      "sim_eres2net_percent": 50.3,
      "dnsmos": 3.81
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0584,
    "raw_value": 5.84,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 48.9% and DNSMOS 3.92 retained as companion metadata only. Row ElevenLabs-v3, evaluated by the Qwen-Audio authors through ElevenLabs' public API (footnote elevenlabs.io/docs/overview/models); voice cloning from the CV3 prompt.",
    "companion": {
      "sim_eres2net_percent": 48.9,
      "dnsmos": 3.92
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.1124,
    "raw_value": 11.24,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 69.1% and DNSMOS 3.79 retained as companion metadata only. Row Qwen3-TTS-12Hz-1.7B-Base, publicly released checkpoint run by the authors. Marked lab: authors (Alibaba Token Foundry) and Qwen3-TTS maker (Alibaba Qwen) share a parent company.",
    "companion": {
      "sim_eres2net_percent": 69.1,
      "dnsmos": 3.79
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0653,
    "raw_value": 6.53,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 66.1% and DNSMOS 3.88 retained as companion metadata only. Row Qwen3-TTS-12Hz-1.7B-Base, publicly released checkpoint run by the authors. Marked lab: authors (Alibaba Token Foundry) and Qwen3-TTS maker (Alibaba Qwen) share a parent company.",
    "companion": {
      "sim_eres2net_percent": 66.1,
      "dnsmos": 3.88
    }
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.1175,
    "raw_value": 11.75,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 77.5% and DNSMOS 3.65 retained as companion metadata only. Row Dots.TTS-2B (SOAR), publicly released checkpoint run by the Qwen-Audio authors.",
    "companion": {
      "sim_eres2net_percent": 77.5,
      "dnsmos": 3.65
    }
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.1169,
    "raw_value": 11.69,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 76.0% and DNSMOS 3.72 retained as companion metadata only. Row Dots.TTS-2B (SOAR), publicly released checkpoint run by the Qwen-Audio authors.",
    "companion": {
      "sim_eres2net_percent": 76,
      "dnsmos": 3.72
    }
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.081,
    "raw_value": 8.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 69.9% and DNSMOS 3.63 retained as companion metadata only. Row VoxCPM2, publicly released checkpoint run by the Qwen-Audio authors.",
    "companion": {
      "sim_eres2net_percent": 69.9,
      "dnsmos": 3.63
    }
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0748,
    "raw_value": 7.48,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 67.0% and DNSMOS 3.73 retained as companion metadata only. Row VoxCPM2, publicly released checkpoint run by the Qwen-Audio authors.",
    "companion": {
      "sim_eres2net_percent": 67,
      "dnsmos": 3.73
    }
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0744,
    "raw_value": 7.44,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer). ERes2Net SIM 78.7% and DNSMOS 3.93 retained as companion metadata only. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used.",
    "companion": {
      "sim_eres2net_percent": 78.7,
      "dnsmos": 3.93
    }
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0671,
    "raw_value": 6.71,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3). ERes2Net SIM 76.6% and DNSMOS 4.04 retained as companion metadata only. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used.",
    "companion": {
      "sim_eres2net_percent": 76.6,
      "dnsmos": 4.04
    }
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0084,
    "raw_value": 0.84,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 3, own row, SEED-TTS-Eval test-zh CER (Paraformer). Baseline rows in Table 3 are cited from other reports and are not imported. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used."
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0154,
    "raw_value": 1.54,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 3, own row, SEED-TTS-Eval test-en WER (Whisper-large-v3). Baseline rows cited from other reports are not imported. Row Qwen-Audio-3.0-TTS (own model). Paper: 'The provider label Qwen-Audio-3.0-TTS-Plus corresponds to the model reported as Qwen-Audio-3.0-TTS in this paper.' The paper's Artificial Analysis Elo claim is NOT used."
  },
  {
    "model": "zonos2-8b",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.093367,
    "raw_value": 9.336667,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row ZONOS2 8B (own model, default mode; the optional 'Quality Mode' row is recorded in quality_mode_components, not as a separate result).",
    "components": {
      "en": 2.76,
      "zh": 15.62,
      "de": 5.67,
      "es": 4.78,
      "fr": 13.18,
      "it": 6.53,
      "ja": 8.27,
      "ko": 18.96,
      "ru": 8.26
    },
    "quality_mode_components": {
      "wer": {
        "en": 3.99,
        "hard_en": 2.68,
        "zh": 6.73,
        "hard_zh": 14.41,
        "de": 3.74,
        "es": 3.25,
        "fr": 4.3,
        "it": 3.52,
        "ja": 7.67,
        "ko": 4.14,
        "ru": 6.99
      },
      "utmos": {
        "en": 3.47,
        "hard_en": 2.94,
        "zh": 3.21,
        "hard_zh": 2.65,
        "de": 3.36,
        "es": 2.94,
        "fr": 3.31,
        "it": 2.92,
        "ja": 3.31,
        "ko": 3.18,
        "ru": 3.02
      }
    }
  },
  {
    "model": "zonos2-8b",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.532778,
    "raw_value": 3.1311,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row ZONOS2 8B (own model, default mode; the optional 'Quality Mode' row is recorded in quality_mode_components, not as a separate result).",
    "components": {
      "en": 3.4,
      "zh": 3.1,
      "de": 3.23,
      "es": 2.96,
      "fr": 3.27,
      "it": 2.89,
      "ja": 3.23,
      "ko": 3.13,
      "ru": 2.97
    }
  },
  {
    "model": "zonos2-8b",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.06144,
    "raw_value": 6.144,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row ZONOS2 8B (own model, default mode; the optional 'Quality Mode' row is recorded in quality_mode_components, not as a separate result).",
    "components": {
      "en": 4.7,
      "zh": 3.19,
      "de": 5.84,
      "es": 5.38,
      "fr": 4.56,
      "it": 5.61,
      "ja": 10.18,
      "ko": 7.45,
      "pt": 6.73,
      "ru": 7.8
    },
    "quality_mode_components": {
      "wer": {
        "en": 2.21,
        "zh": 2.77,
        "ar": 13.94,
        "de": 3.37,
        "es": 2.1,
        "fr": 2.79,
        "hi": 9.04,
        "id": 6.48,
        "it": 2.51,
        "ja": 8.7,
        "ko": 4.59,
        "pl": 5.85,
        "pt": 2.94,
        "ru": 4.32,
        "th": 5.93,
        "tl": 16.05,
        "tr": 7.47
      },
      "utmos": {
        "en": 2.99,
        "zh": 2.68,
        "ar": 2.51,
        "de": 2.92,
        "es": 2.72,
        "fr": 2.74,
        "hi": 2.73,
        "id": 2.66,
        "it": 2.63,
        "ja": 2.69,
        "ko": 2.78,
        "pl": 2.76,
        "pt": 2.75,
        "ru": 2.62,
        "th": 2.74,
        "tl": 2.88,
        "tr": 2.64
      }
    }
  },
  {
    "model": "zonos2-8b",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.3505,
    "raw_value": 2.402,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row ZONOS2 8B (own model, default mode; the optional 'Quality Mode' row is recorded in quality_mode_components, not as a separate result).",
    "components": {
      "en": 2.44,
      "zh": 2.43,
      "de": 2.52,
      "es": 2.42,
      "fr": 2.37,
      "it": 2.32,
      "ja": 2.34,
      "ko": 2.55,
      "pt": 2.33,
      "ru": 2.3
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.028622,
    "raw_value": 2.862222,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row 'Qwen 3 TTS 1.7B' (Hu et al. 2026, Qwen3-TTS report), run with reference-audio zero-shot cloning (speaker similarity not starred). The only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base.",
    "components": {
      "en": 1.94,
      "zh": 2.91,
      "de": 2.78,
      "es": 1.98,
      "fr": 3.08,
      "it": 2.19,
      "ja": 3.78,
      "ko": 2.85,
      "ru": 4.25
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.651944,
    "raw_value": 3.6078,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row 'Qwen 3 TTS 1.7B' (Hu et al. 2026, Qwen3-TTS report), run with reference-audio zero-shot cloning (speaker similarity not starred). The only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base.",
    "components": {
      "en": 3.86,
      "zh": 3.71,
      "de": 3.72,
      "es": 3.43,
      "fr": 3.64,
      "it": 3.38,
      "ja": 3.71,
      "ko": 3.53,
      "ru": 3.49
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.02259,
    "raw_value": 2.259,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row 'Qwen 3 TTS 1.7B' (Hu et al. 2026, Qwen3-TTS report), run with reference-audio zero-shot cloning (speaker similarity not starred). The only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base.",
    "components": {
      "en": 1.05,
      "zh": 0.99,
      "de": 2.11,
      "es": 1.77,
      "fr": 2.62,
      "it": 2.85,
      "ja": 2.32,
      "ko": 2.8,
      "pt": 2.2,
      "ru": 3.88
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.47775,
    "raw_value": 2.911,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row 'Qwen 3 TTS 1.7B' (Hu et al. 2026, Qwen3-TTS report), run with reference-audio zero-shot cloning (speaker similarity not starred). The only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base.",
    "components": {
      "en": 3.2,
      "zh": 2.9,
      "de": 3.12,
      "es": 2.87,
      "fr": 2.84,
      "it": 2.76,
      "ja": 2.86,
      "ko": 2.97,
      "pt": 2.85,
      "ru": 2.74
    }
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.045144,
    "raw_value": 4.514444,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row Fish S2 Pro (Liao et al. 2026).",
    "components": {
      "en": 3.6,
      "zh": 4.33,
      "de": 3.04,
      "es": 3.73,
      "fr": 5.9,
      "it": 4.17,
      "ja": 4.2,
      "ko": 3.92,
      "ru": 7.74
    }
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.551389,
    "raw_value": 3.2056,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Fish S2 Pro (Liao et al. 2026).",
    "components": {
      "en": 3.47,
      "zh": 3.23,
      "de": 3.26,
      "es": 3.01,
      "fr": 3.22,
      "it": 3.02,
      "ja": 3.37,
      "ko": 3.18,
      "ru": 3.09
    }
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.029,
    "raw_value": 2.9,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row Fish S2 Pro (Liao et al. 2026).",
    "components": {
      "en": 2.09,
      "zh": 1.26,
      "de": 2.65,
      "es": 2.97,
      "fr": 3.45,
      "it": 2.79,
      "ja": 2.17,
      "ko": 3.48,
      "pt": 2.86,
      "ru": 5.28
    }
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.42675,
    "raw_value": 2.707,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Fish S2 Pro (Liao et al. 2026).",
    "components": {
      "en": 2.92,
      "zh": 2.73,
      "de": 2.9,
      "es": 2.69,
      "fr": 2.67,
      "it": 2.54,
      "ja": 2.73,
      "ko": 2.76,
      "pt": 2.57,
      "ru": 2.56
    }
  },
  {
    "model": "voxcpm2",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.050844,
    "raw_value": 5.084444,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row VoxCPM 2 (Zhou et al. 2026, VoxCPM2 report).",
    "components": {
      "en": 4.23,
      "zh": 5.01,
      "de": 4.84,
      "es": 4.44,
      "fr": 5.1,
      "it": 5.8,
      "ja": 4.92,
      "ko": 3.99,
      "ru": 7.43
    }
  },
  {
    "model": "voxcpm2",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.516944,
    "raw_value": 3.0678,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row VoxCPM 2 (Zhou et al. 2026, VoxCPM2 report).",
    "components": {
      "en": 3.51,
      "zh": 3.12,
      "de": 3.07,
      "es": 2.85,
      "fr": 3.25,
      "it": 2.71,
      "ja": 3.23,
      "ko": 3.04,
      "ru": 2.83
    }
  },
  {
    "model": "voxcpm2",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.03611,
    "raw_value": 3.611,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row VoxCPM 2 (Zhou et al. 2026, VoxCPM2 report).",
    "components": {
      "en": 1.69,
      "zh": 1.44,
      "de": 2.84,
      "es": 3.19,
      "fr": 4.54,
      "it": 3.5,
      "ja": 4.56,
      "ko": 4.89,
      "pt": 4.09,
      "ru": 5.37
    }
  },
  {
    "model": "voxcpm2",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.34875,
    "raw_value": 2.395,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row VoxCPM 2 (Zhou et al. 2026, VoxCPM2 report).",
    "components": {
      "en": 2.51,
      "zh": 2.4,
      "de": 2.6,
      "es": 2.39,
      "fr": 2.36,
      "it": 2.26,
      "ja": 2.3,
      "ko": 2.51,
      "pt": 2.32,
      "ru": 2.3
    }
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.036444,
    "raw_value": 3.644444,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row Cartesia Sonic 3.5 (reference: cartesia.ai/sonic, accessed 2026-06-03). Sonic 3.5 is NOT Sonic 3.6.",
    "components": {
      "en": 2.56,
      "zh": 4.53,
      "de": 3.14,
      "es": 3.21,
      "fr": 3.41,
      "it": 3.05,
      "ja": 3.65,
      "ko": 3.24,
      "ru": 6.01
    }
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.545833,
    "raw_value": 3.1833,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Cartesia Sonic 3.5 (reference: cartesia.ai/sonic, accessed 2026-06-03). Sonic 3.5 is NOT Sonic 3.6.",
    "components": {
      "en": 3.62,
      "zh": 3.17,
      "de": 3.18,
      "es": 2.97,
      "fr": 3.29,
      "it": 2.9,
      "ja": 3.36,
      "ko": 3.11,
      "ru": 3.05
    }
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.02572,
    "raw_value": 2.572,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row Cartesia Sonic 3.5 (reference: cartesia.ai/sonic, accessed 2026-06-03). Sonic 3.5 is NOT Sonic 3.6.",
    "components": {
      "en": 1.4,
      "zh": 1.17,
      "de": 3,
      "es": 2.37,
      "fr": 2.52,
      "it": 2.4,
      "ja": 3.63,
      "ko": 3.38,
      "pt": 2.1,
      "ru": 3.75
    }
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.4415,
    "raw_value": 2.766,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Cartesia Sonic 3.5 (reference: cartesia.ai/sonic, accessed 2026-06-03). Sonic 3.5 is NOT Sonic 3.6.",
    "components": {
      "en": 3.05,
      "zh": 2.76,
      "de": 2.98,
      "es": 2.71,
      "fr": 2.69,
      "it": 2.6,
      "ja": 2.69,
      "ko": 2.86,
      "pt": 2.68,
      "ru": 2.64
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.038178,
    "raw_value": 3.817778,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row Eleven Labs V3 (reference elevenlabs.io/v3, accessed 2026-06-03); starred = no zero-shot cloning, preset voice.",
    "components": {
      "en": 2.35,
      "zh": 4.2,
      "de": 3.79,
      "es": 3.24,
      "fr": 3.62,
      "it": 2.7,
      "ja": 4.26,
      "ko": 4.57,
      "ru": 5.63
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.586944,
    "raw_value": 3.3478,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Eleven Labs V3 (reference elevenlabs.io/v3, accessed 2026-06-03); starred = no zero-shot cloning, preset voice.",
    "components": {
      "en": 3.59,
      "zh": 3.31,
      "de": 3.54,
      "es": 3.25,
      "fr": 3.29,
      "it": 3.19,
      "ja": 3.45,
      "ko": 3.36,
      "ru": 3.15
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.02429,
    "raw_value": 2.429,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row Eleven Labs V3 (reference elevenlabs.io/v3, accessed 2026-06-03); starred = no zero-shot cloning, preset voice.",
    "components": {
      "en": 1.35,
      "zh": 1.22,
      "de": 2.5,
      "es": 1.58,
      "fr": 2.17,
      "it": 2.36,
      "ja": 2.84,
      "ko": 3.93,
      "pt": 2.06,
      "ru": 4.28
    }
  },
  {
    "model": "elevenlabs-v3",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.59775,
    "raw_value": 3.391,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Eleven Labs V3 (reference elevenlabs.io/v3, accessed 2026-06-03); starred = no zero-shot cloning, preset voice.",
    "components": {
      "en": 3.61,
      "zh": 3.36,
      "de": 3.57,
      "es": 3.34,
      "fr": 3.28,
      "it": 3.21,
      "ja": 3.49,
      "ko": 3.4,
      "pt": 3.39,
      "ru": 3.26
    }
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.040056,
    "raw_value": 4.005556,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row 'Gemini 3.1 Flash', citing the Gemini 3.1 Flash Audio model card (covers Gemini 3.1 Flash TTS; accessed 2026-06-03); the only Gemini 3.1 Flash TTS API id is gemini-3.1-flash-tts-preview. Starred = preset voice, no cloning.",
    "components": {
      "en": 2.5,
      "zh": 4.09,
      "de": 3.61,
      "es": 4.41,
      "fr": 4.77,
      "it": 3.24,
      "ja": 3.79,
      "ko": 3.06,
      "ru": 6.58
    }
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.638611,
    "raw_value": 3.5544,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row 'Gemini 3.1 Flash', citing the Gemini 3.1 Flash Audio model card (covers Gemini 3.1 Flash TTS; accessed 2026-06-03); the only Gemini 3.1 Flash TTS API id is gemini-3.1-flash-tts-preview. Starred = preset voice, no cloning.",
    "components": {
      "en": 3.87,
      "zh": 3.54,
      "de": 3.71,
      "es": 3.43,
      "fr": 3.52,
      "it": 3.39,
      "ja": 3.65,
      "ko": 3.51,
      "ru": 3.37
    }
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.02552,
    "raw_value": 2.552,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row 'Gemini 3.1 Flash', citing the Gemini 3.1 Flash Audio model card (covers Gemini 3.1 Flash TTS; accessed 2026-06-03); the only Gemini 3.1 Flash TTS API id is gemini-3.1-flash-tts-preview. Starred = preset voice, no cloning.",
    "components": {
      "en": 2.12,
      "zh": 1.45,
      "de": 2.4,
      "es": 1.94,
      "fr": 3.56,
      "it": 2.08,
      "ja": 2.72,
      "ko": 2.27,
      "pt": 3.21,
      "ru": 3.77
    }
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.62775,
    "raw_value": 3.511,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row 'Gemini 3.1 Flash', citing the Gemini 3.1 Flash Audio model card (covers Gemini 3.1 Flash TTS; accessed 2026-06-03); the only Gemini 3.1 Flash TTS API id is gemini-3.1-flash-tts-preview. Starred = preset voice, no cloning.",
    "components": {
      "en": 3.78,
      "zh": 3.55,
      "de": 3.67,
      "es": 3.41,
      "fr": 3.45,
      "it": 3.45,
      "ja": 3.56,
      "ko": 3.45,
      "pt": 3.43,
      "ru": 3.36
    }
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "ztts1-clean-ml9-wer",
    "score": 0.044867,
    "raw_value": 4.486667,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru (languages every compared system supports; hard/unsupported columns excluded). Row Inworld TTS 2, citing 'Inworld AI Realtime TTS-2' (inworld.ai/blog/realtime-tts-2, accessed 2026-06-03); API id inworld-tts-2. Evaluated before the 2026-08-31 GA post; backend build may differ from GA.",
    "components": {
      "en": 3.14,
      "zh": 4.04,
      "de": 4.05,
      "es": 4.38,
      "fr": 4.95,
      "it": 3.98,
      "ja": 5.63,
      "ko": 3.13,
      "ru": 7.08
    }
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "ztts1-clean-ml9-utmos",
    "score": 0.546111,
    "raw_value": 3.1844,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table III (ZTTS1-Eval Clean, FLEURS-R read speech), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Inworld TTS 2, citing 'Inworld AI Realtime TTS-2' (inworld.ai/blog/realtime-tts-2, accessed 2026-06-03); API id inworld-tts-2. Evaluated before the 2026-08-31 GA post; backend build may differ from GA.",
    "components": {
      "en": 3.53,
      "zh": 3.14,
      "de": 3.27,
      "es": 3.05,
      "fr": 3.33,
      "it": 2.95,
      "ja": 3.27,
      "ko": 3.11,
      "ru": 3.01
    }
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "ztts1-itw-10lang-wer",
    "score": 0.03766,
    "raw_value": 3.766,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), WER column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru (languages every compared system supports; hard/unsupported columns excluded). Row Inworld TTS 2, citing 'Inworld AI Realtime TTS-2' (inworld.ai/blog/realtime-tts-2, accessed 2026-06-03); API id inworld-tts-2. Evaluated before the 2026-08-31 GA post; backend build may differ from GA.",
    "components": {
      "en": 1.89,
      "zh": 1.42,
      "de": 3.54,
      "es": 2.67,
      "fr": 4.4,
      "it": 3.46,
      "ja": 6.93,
      "ko": 4.16,
      "pt": 3.87,
      "ru": 5.32
    }
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "ztts1-itw-10lang-utmos",
    "score": 0.45225,
    "raw_value": 2.809,
    "raw_unit": "MSR-UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "independent",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). ZTTS1-Eval run by Zyphra: Qwen3-ASR scorer, MSR-UTMOS, ReDimNet. Table IV (ZTTS1-Eval in-the-wild, VoxBlink2 prompts), UTMOS column. Derived unweighted mean over en,zh,de,es,fr,it,ja,ko,pt,ru; normalized (UTMOS-1)/4. Automatic MOS predictor, not human MOS. Row Inworld TTS 2, citing 'Inworld AI Realtime TTS-2' (inworld.ai/blog/realtime-tts-2, accessed 2026-06-03); API id inworld-tts-2. Evaluated before the 2026-08-31 GA post; backend build may differ from GA.",
    "components": {
      "en": 3.02,
      "zh": 2.71,
      "de": 3.03,
      "es": 2.83,
      "fr": 2.76,
      "it": 2.71,
      "ja": 2.81,
      "ko": 2.86,
      "pt": 2.7,
      "ru": 2.66
    }
  },
  {
    "model": "zonos2-8b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0205,
    "raw_value": 2.05,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). Appendix Table VII, own ZONOS2 Seed-TTS-Eval zero-shot test-en WER (Seed-TTS-Eval scoring stack Whisper-large / Paraformer per Table II). Own model, default mode."
  },
  {
    "model": "zonos2-8b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0255,
    "raw_value": 2.55,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2606.24320v2",
    "source_id": "speech-zonos2-report-bysa",
    "by": "lab",
    "date": "2026-06-25",
    "note": "ZONOS2 Technical Report (Zyphra), arXiv:2606.24320v2, CC BY-SA 4.0 (share-alike). Appendix Table VII, own ZONOS2 Seed-TTS-Eval zero-shot test-zh error (CER) (Seed-TTS-Eval scoring stack Whisper-large / Paraformer per Table II). Own model, default mode."
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "rw-voice-eq-tts-acting",
    "score": 0.7425,
    "raw_value": 3.97,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acting / Role-Fit column, exact printed cell 3.97 ± 0.04 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-3.1-flash-tts-preview, voice Kore.",
    "figure_rank": 1
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "rw-voice-eq-tts-acting",
    "score": 0.66,
    "raw_value": 3.64,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acting / Role-Fit column, exact printed cell 3.64 ± 0.03 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-flash-preview-tts, voice Kore.",
    "figure_rank": 2
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "rw-voice-eq-tts-acting",
    "score": 0.6575,
    "raw_value": 3.63,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acting / Role-Fit column, exact printed cell 3.63 ± 0.04 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-pro-preview-tts, voice Kore.",
    "figure_rank": 3
  },
  {
    "model": "openai-tts-1",
    "bench": "rw-voice-eq-tts-acting",
    "score": 0.54,
    "raw_value": 3.16,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acting / Role-Fit column, exact printed cell 3.16 ± 0.02 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API model tts-1, voice alloy.",
    "figure_rank": 4
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "rw-voice-eq-tts-expressiveness",
    "score": 0.66,
    "raw_value": 3.64,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Expressiveness column, exact printed cell 3.64 ± 0.04 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-3.1-flash-tts-preview, voice Kore.",
    "figure_rank": 1
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "rw-voice-eq-tts-expressiveness",
    "score": 0.61,
    "raw_value": 3.44,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Expressiveness column, exact printed cell 3.44 ± 0.03 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-flash-preview-tts, voice Kore.",
    "figure_rank": 2
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "rw-voice-eq-tts-expressiveness",
    "score": 0.58,
    "raw_value": 3.32,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Expressiveness column, exact printed cell 3.32 ± 0.04 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-pro-preview-tts, voice Kore.",
    "figure_rank": 3
  },
  {
    "model": "smallest-lightning-v3-1",
    "bench": "rw-voice-eq-tts-expressiveness",
    "score": 0.575,
    "raw_value": 3.3,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Expressiveness column, exact printed cell 3.30 ± 0.03 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: Smallest.ai Lightning, API lightning_v3.1, voice avery (release 2026-03-27). NOT Lightning v3.1 Pro.",
    "figure_rank": 4
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "rw-voice-eq-tts-expressiveness",
    "score": 0.5575,
    "raw_value": 3.23,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Expressiveness column, exact printed cell 3.23 ± 0.03 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gpt-4o-mini-tts, voice alloy.",
    "figure_rank": 5
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "rw-voice-eq-tts-voice-identity",
    "score": 0.6325,
    "raw_value": 3.53,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Voice Identity column, exact printed cell 3.53 ± 0.02 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API sonic-3.5, voice katie (release 2026-06-16). NOT Sonic 3.6.",
    "figure_rank": 1
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rw-voice-eq-tts-voice-identity",
    "score": 0.6325,
    "raw_value": 3.53,
    "raw_sd": 0.01,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Voice Identity column, exact printed cell 3.53 ± 0.01 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Figure label 'Fish Audio S2'; Appendix Table 4 lists Fish Speech as fishaudio/s2-pro (reference voice, 2026-03-09) and a Replicate deployment jichengdu/fish-speech with the same S2 release date; which deployment produced this cell is not stated.",
    "figure_rank": 2
  },
  {
    "model": "kokoro-82m",
    "bench": "rw-voice-eq-tts-voice-identity",
    "score": 0.625,
    "raw_value": 3.5,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Voice Identity column, exact printed cell 3.50 ± 0.02 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: hexgrad/Kokoro-82M, voice af_heart (a v1.0 voice); evaluated through local, Modal and Replicate deployments, figure shows one Kokoro-82M row.",
    "figure_rank": 3
  },
  {
    "model": "openai-tts-1",
    "bench": "rw-voice-eq-tts-voice-identity",
    "score": 0.615,
    "raw_value": 3.46,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Voice Identity column, exact printed cell 3.46 ± 0.03 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API model tts-1, voice alloy.",
    "figure_rank": 4
  },
  {
    "model": "openai-tts-1-hd",
    "bench": "rw-voice-eq-tts-voice-identity",
    "score": 0.615,
    "raw_value": 3.46,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Voice Identity column, exact printed cell 3.46 ± 0.04 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API model tts-1-hd, voice alloy.",
    "figure_rank": 5
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rw-voice-eq-tts-language-switching",
    "score": 0.62,
    "raw_value": 3.48,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Language Code-Switching column, exact printed cell 3.48 ± 0.02 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Figure label 'Fish Audio S2'; Appendix Table 4 lists Fish Speech as fishaudio/s2-pro (reference voice, 2026-03-09) and a Replicate deployment jichengdu/fish-speech with the same S2 release date; which deployment produced this cell is not stated.",
    "figure_rank": 2
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "rw-voice-eq-tts-language-switching",
    "score": 0.595,
    "raw_value": 3.38,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Language Code-Switching column, exact printed cell 3.38 ± 0.03 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: bosonai/higgs-audio-v3-tts-4b, server default voice.",
    "figure_rank": 3
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "rw-voice-eq-tts-language-switching",
    "score": 0.585,
    "raw_value": 3.34,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Language Code-Switching column, exact printed cell 3.34 ± 0.03 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gpt-4o-mini-tts, voice alloy.",
    "figure_rank": 4
  },
  {
    "model": "elevenlabs-v3",
    "bench": "rw-voice-eq-tts-language-switching",
    "score": 0.5825,
    "raw_value": 3.33,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Language Code-Switching column, exact printed cell 3.33 ± 0.04 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: eleven_v3, voice rachel; standard TTS interface (a separate voice-design configuration exists).",
    "figure_rank": 5
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "rw-voice-eq-tts-reliability",
    "score": 0.7225,
    "raw_value": 3.89,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Reliability column, exact printed cell 3.89 ± 0.04 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-pro-preview-tts, voice Kore.",
    "figure_rank": 1
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "rw-voice-eq-tts-reliability",
    "score": 0.705,
    "raw_value": 3.82,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Reliability column, exact printed cell 3.82 ± 0.04 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-3.1-flash-tts-preview, voice Kore.",
    "figure_rank": 2
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "rw-voice-eq-tts-reliability",
    "score": 0.665,
    "raw_value": 3.66,
    "raw_sd": 0.05,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Reliability column, exact printed cell 3.66 ± 0.05 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gemini-2.5-flash-preview-tts, voice Kore.",
    "figure_rank": 3
  },
  {
    "model": "deepgram-aura-2",
    "bench": "rw-voice-eq-tts-reliability",
    "score": 0.65,
    "raw_value": 3.6,
    "raw_sd": 0.05,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Reliability column, exact printed cell 3.60 ± 0.05 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: aura-2-thalia-en, voice thalia.",
    "figure_rank": 4
  },
  {
    "model": "elevenlabs-v3",
    "bench": "rw-voice-eq-tts-reliability",
    "score": 0.6475,
    "raw_value": 3.59,
    "raw_sd": 0.05,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Reliability column, exact printed cell 3.59 ± 0.05 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: eleven_v3, voice rachel; standard TTS interface (a separate voice-design configuration exists).",
    "figure_rank": 5
  },
  {
    "model": "kokoro-82m",
    "bench": "rw-voice-eq-tts-longform-stability",
    "score": 0.635,
    "raw_value": 3.54,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Long-Form Stability column, exact printed cell 3.54 ± 0.04 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: hexgrad/Kokoro-82M, voice af_heart (a v1.0 voice); evaluated through local, Modal and Replicate deployments, figure shows one Kokoro-82M row.",
    "figure_rank": 1
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "rw-voice-eq-tts-longform-stability",
    "score": 0.6325,
    "raw_value": 3.53,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Long-Form Stability column, exact printed cell 3.53 ± 0.04 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API sonic-3.5, voice katie (release 2026-06-16). NOT Sonic 3.6.",
    "figure_rank": 2
  },
  {
    "model": "indextts-2",
    "bench": "rw-voice-eq-tts-longform-stability",
    "score": 0.63,
    "raw_value": 3.52,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Long-Form Stability column, exact printed cell 3.52 ± 0.04 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: IndexTTS2 via Replicate lucataco/indextts-2, reference voice.",
    "figure_rank": 3
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rw-voice-eq-tts-longform-stability",
    "score": 0.625,
    "raw_value": 3.5,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Long-Form Stability column, exact printed cell 3.50 ± 0.02 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Figure label 'Fish Audio S2'; Appendix Table 4 lists Fish Speech as fishaudio/s2-pro (reference voice, 2026-03-09) and a Replicate deployment jichengdu/fish-speech with the same S2 release date; which deployment produced this cell is not stated.",
    "figure_rank": 4
  },
  {
    "model": "sesame-csm-1b",
    "bench": "rw-voice-eq-tts-longform-stability",
    "score": 0.61,
    "raw_value": 3.44,
    "raw_sd": 0.04,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Long-Form Stability column, exact printed cell 3.44 ± 0.04 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: CSM-1B via Replicate lucataco/csm-1b, speaker 0, release 2025-03-13 (Sesame CSM-1B; id defined by the TTS-commercial lane).",
    "figure_rank": 5
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "rw-voice-eq-tts-acoustic-quality",
    "score": 0.5925,
    "raw_value": 3.37,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acoustic Quality column, exact printed cell 3.37 ± 0.02 (within-capability rank 1). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API sonic-3.5, voice katie (release 2026-06-16). NOT Sonic 3.6.",
    "figure_rank": 1
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rw-voice-eq-tts-acoustic-quality",
    "score": 0.585,
    "raw_value": 3.34,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acoustic Quality column, exact printed cell 3.34 ± 0.02 (within-capability rank 2). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Figure label 'Fish Audio S2'; Appendix Table 4 lists Fish Speech as fishaudio/s2-pro (reference voice, 2026-03-09) and a Replicate deployment jichengdu/fish-speech with the same S2 release date; which deployment produced this cell is not stated.",
    "figure_rank": 2
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "rw-voice-eq-tts-acoustic-quality",
    "score": 0.585,
    "raw_value": 3.34,
    "raw_sd": 0.03,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acoustic Quality column, exact printed cell 3.34 ± 0.03 (within-capability rank 3). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: gpt-4o-mini-tts, voice alloy.",
    "figure_rank": 3
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "rw-voice-eq-tts-acoustic-quality",
    "score": 0.5825,
    "raw_value": 3.33,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acoustic Quality column, exact printed cell 3.33 ± 0.02 (within-capability rank 4). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API inworld-tts-2, voice Ashley (listed release 2026-05-05). Same API id later announced GA as Realtime TTS-2 (2026-08-31); backend build may differ.",
    "figure_rank": 4
  },
  {
    "model": "openai-tts-1",
    "bench": "rw-voice-eq-tts-acoustic-quality",
    "score": 0.5825,
    "raw_value": 3.33,
    "raw_sd": 0.02,
    "raw_unit": "rater-controlled factor score on the 1-5 rating scale",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 5 (TTS top-five matrix), Acoustic Quality column, exact printed cell 3.33 ± 0.02 (within-capability rank 5). Hume human raters (US/UK/CA native English; language-matched for code-switching), provider default voice/settings, 3 generations per prompt. Only the top five per dimension are published; normalized (score-1)/4. Appendix Table 4: API model tts-1, voice alloy.",
    "figure_rank": 5
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0672,
    "raw_value": 6.72,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 6.72% for elevenlabs/scribe-v2. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0743,
    "raw_value": 7.43,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 7.43% for qwen/qwen3-asr-1.7b. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0834,
    "raw_value": 8.34,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 8.34% for nvidia/parakeet-tdt-0.6b-v3. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "google-chirp-3",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.087,
    "raw_value": 8.7,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 8.70% for google/chirp_3. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0904,
    "raw_value": 9.04,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 9.04% for speechmatics/enhanced. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0918,
    "raw_value": 9.18,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 9.18% for nvidia/canary-qwen-2.5b. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "deepgram-nova-3",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0953,
    "raw_value": 9.53,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 9.53% for deepgram/nova-3. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "distil-large-v3-5",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0965,
    "raw_value": 9.65,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 9.65% for distil-whisper/distil-large-v3.5. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "higgs-audio-v2",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.021,
    "raw_value": 2.1,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Higgs Audio v2 (Boson's own earlier model)."
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.0111,
    "raw_value": 1.11,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Higgs Audio v3 (own model; same numbers as the bosonai/higgs-audio-v3-tts-4b model card, which is NC-licensed and used only for identity corroboration)."
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.0131,
    "raw_value": 1.31,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Fish Audio S2 Pro, run by Boson AI (competitor evaluation)."
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.013,
    "raw_value": 1.3,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column 'Qwen3-TTS-1.7B', run by Boson AI in reference-audio voice-clone benchmarks; the only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base."
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.0163,
    "raw_value": 1.63,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column IndexTTS-2, run by Boson AI."
  },
  {
    "model": "higgs-audio-v2",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.2119,
    "raw_value": 21.19,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column Higgs Audio v2 (Boson's own earlier model)."
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0441,
    "raw_value": 4.41,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Higgs Audio v3 (own model; same numbers as the bosonai/higgs-audio-v3-tts-4b model card, which is NC-licensed and used only for identity corroboration)."
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.046,
    "raw_value": 4.6,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Fish Audio S2 Pro, run by Boson AI (competitor evaluation)."
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0773,
    "raw_value": 7.73,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column 'Qwen3-TTS-1.7B', run by Boson AI in reference-audio voice-clone benchmarks; the only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base."
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 1.2926,
    "raw_value": 129.26,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column IndexTTS-2, run by Boson AI."
  },
  {
    "model": "higgs-audio-v2",
    "bench": "minimax-ml23-macro-err",
    "score": 0.4986,
    "raw_value": 49.86,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column Higgs Audio v2 (Boson's own earlier model)."
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "minimax-ml23-macro-err",
    "score": 0.0274,
    "raw_value": 2.74,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "lab",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Higgs Audio v3 (own model; same numbers as the bosonai/higgs-audio-v3-tts-4b model card, which is NC-licensed and used only for identity corroboration)."
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "minimax-ml23-macro-err",
    "score": 0.0515,
    "raw_value": 5.15,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Column Fish Audio S2 Pro, run by Boson AI (competitor evaluation)."
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "minimax-ml23-macro-err",
    "score": 0.2741,
    "raw_value": 27.41,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column 'Qwen3-TTS-1.7B', run by Boson AI in reference-audio voice-clone benchmarks; the only released 1.7B Qwen3-TTS checkpoint that clones from reference audio is Qwen3-TTS-12Hz-1.7B-Base."
  },
  {
    "model": "indextts-2",
    "bench": "minimax-ml23-macro-err",
    "score": 1.1291,
    "raw_value": 112.91,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Includes languages the model does not officially support; the error ratio reflects coverage as well as quality. Column IndexTTS-2, run by Boson AI."
  },
  {
    "model": "openaudio-s1-mini",
    "bench": "lemas-eval-9lang-wer",
    "score": 0.1227,
    "raw_value": 12.27,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2601.04233v1",
    "source_id": "speech-lemas-report-bysa",
    "by": "independent",
    "date": "2026-01-04",
    "note": "LEMAS report, arXiv:2601.04233v1, CC BY-SA 4.0 (share-alike). Table 4, row OpenAudio-S1-Mini, 'Avg' WER as printed = unweighted mean of the nine languages with a value (de,en,es,fr,id,it,pt,ru,zh; vi blank for S1-mini), recomputed and matched. LEMAS evaluation set: 500 utterances per language; Paraformer-zh for Chinese, Whisper-large-v3 otherwise; baseline run by the LEMAS authors via the fishaudio/openaudio-s1-mini release (paper footnote to the official HF Space). WavLM SIM avg 0.480 kept as companion metadata. Independent: LEMAS authors do not make S1-mini.",
    "components": {
      "de": 3.95,
      "en": 4.98,
      "es": 7.66,
      "fr": 13.41,
      "id": 32.77,
      "it": 9.27,
      "pt": 8.05,
      "ru": 20.75,
      "zh": 9.59
    },
    "companion": {
      "sim_wavlm_avg": 0.48
    }
  },
  {
    "model": "hume-octave",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1285,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row HumeAI (linked hume.ai/research); strongly prompted (star). Identity: the CC BY v1 paper (arXiv:2505.23009v1, submitted 2025-05-29) names the benchmarked closed-source system 'HumeAI's Octave (TTS)'; api_clients.py HumeAIClient calls tts.synthesize_json without a version parameter. README per-category WERs (0.83/21.05/19.84/37.14/0.38/0.93, overall 12.85) equal the paper's HumeAI row, so the README re-judged the May 2025 generations. Octave 2 was first previewed 2025-10-01 (hume.ai/blog/octave-2-launch), so these clips are Octave 1. Hume generates a different voice per utterance (paper §4.4). Stable Gemini judge six-category overall column.",
    "raw_value": 12.85,
    "scale_divisor": 100,
    "voice": "–",
    "strong_prompting": true,
    "source_model_label": "HumeAI",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/api_clients.py"
    ]
  },
  {
    "model": "hume-octave",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.4677,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row HumeAI (linked hume.ai/research); strongly prompted (star). Identity: the CC BY v1 paper (arXiv:2505.23009v1, submitted 2025-05-29) names the benchmarked closed-source system 'HumeAI's Octave (TTS)'; api_clients.py HumeAIClient calls tts.synthesize_json without a version parameter. README per-category WERs (0.83/21.05/19.84/37.14/0.38/0.93, overall 12.85) equal the paper's HumeAI row, so the README re-judged the May 2025 generations. Octave 2 was first previewed 2025-10-01 (hume.ai/blog/octave-2-launch), so these clips are Octave 1. Hume generates a different voice per utterance (paper §4.4). Stable Gemini judge six-category overall column.",
    "raw_value": 46.77,
    "scale_divisor": 100,
    "voice": "–",
    "strong_prompting": true,
    "source_model_label": "HumeAI",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/api_clients.py"
    ]
  },
  {
    "model": "orpheus-tts-0-1-finetune-prod",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.1771,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Orpheus TTS; voice Tara; no strong-prompting star. evaluation_runner.py maps HF checkpoint canopylabs/orpheus-tts-0.1-finetune-prod to OrpheusClient (voice default 'tara'). Stable Gemini judge six-category overall column. Weight revision not pinned.",
    "raw_value": 17.71,
    "scale_divisor": 100,
    "voice": "Tara",
    "strong_prompting": false,
    "source_model_label": "Orpheus TTS",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/model_clients.py"
    ]
  },
  {
    "model": "orpheus-tts-0-1-finetune-prod",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.2944,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Orpheus TTS; voice Tara; no strong-prompting star. evaluation_runner.py maps HF checkpoint canopylabs/orpheus-tts-0.1-finetune-prod to OrpheusClient (voice default 'tara'). Stable Gemini judge six-category overall column. Weight revision not pinned.",
    "raw_value": 29.44,
    "scale_divisor": 100,
    "voice": "Tara",
    "strong_prompting": false,
    "source_model_label": "Orpheus TTS",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/model_clients.py"
    ]
  },
  {
    "model": "sesame-csm-1b",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.3232,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Sesame1B (linked SesameAILabs/csm); no strong-prompting star. evaluation_runner.py/model_clients.py load it via generator.load_csm_1b (Sesame CSM-1B), speaker=0, no context. Stable Gemini judge six-category overall column. Weight revision not pinned.",
    "raw_value": 32.32,
    "scale_divisor": 100,
    "voice": "–",
    "strong_prompting": false,
    "source_model_label": "Sesame1B",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/model_clients.py"
    ]
  },
  {
    "model": "sesame-csm-1b",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.1337,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Sesame1B (linked SesameAILabs/csm); no strong-prompting star. evaluation_runner.py/model_clients.py load it via generator.load_csm_1b (Sesame CSM-1B), speaker=0, no context. Stable Gemini judge six-category overall column. Weight revision not pinned.",
    "raw_value": 13.37,
    "scale_divisor": 100,
    "voice": "–",
    "strong_prompting": false,
    "source_model_label": "Sesame1B",
    "identity_sources": [
      "https://arxiv.org/abs/2505.23009v1",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/evaluation_runner.py",
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/model_clients.py"
    ]
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.8,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'gpt-4o-mini-tts †'; voice Alloy; strong prompting (†). MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 10.76. Raw MOS 4.2; normalized (MOS-1)/4.",
    "raw_value": 4.2,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "Alloy",
    "strong_prompting": true,
    "source_model_label": "gpt-4o-mini-tts †"
  },
  {
    "model": "zonos-v0-1",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.5975,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'Zyphra/Zonos'; voice exampleaudio; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 19.12. Raw MOS 3.39; normalized (MOS-1)/4.",
    "raw_value": 3.39,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "exampleaudio",
    "strong_prompting": false,
    "source_model_label": "Zyphra/Zonos"
  },
  {
    "model": "deepgram-aura-2",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.8325,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'DeepGram Aura-2'; voice Thalia-en; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 16.83. Raw MOS 4.33; normalized (MOS-1)/4.",
    "raw_value": 4.33,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "Thalia-en",
    "strong_prompting": false,
    "source_model_label": "DeepGram Aura-2"
  },
  {
    "model": "elevenlabs-multilingual-v2",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.6375,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row '11Labs eleven multilingual v2'; voice Brian; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 11.19. Raw MOS 3.55; normalized (MOS-1)/4.",
    "raw_value": 3.55,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "Brian",
    "strong_prompting": false,
    "source_model_label": "11Labs eleven multilingual v2"
  },
  {
    "model": "hume-octave",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.795,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'HumeAI †'; voice –; strong prompting (†). MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 12.85. Raw MOS 4.18; normalized (MOS-1)/4. Identity: paper names the system 'HumeAI's Octave (TTS)' (May 2025, before Octave 2).",
    "raw_value": 4.18,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "–",
    "strong_prompting": true,
    "source_model_label": "HumeAI †"
  },
  {
    "model": "orpheus-tts-0-1-finetune-prod",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.69,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'Orpheus TTS'; voice Tara; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 17.71. Raw MOS 3.76; normalized (MOS-1)/4.",
    "raw_value": 3.76,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "Tara",
    "strong_prompting": false,
    "source_model_label": "Orpheus TTS"
  },
  {
    "model": "sesame-csm-1b",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.595,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'Sesame1B'; voice –; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 32.32. Raw MOS 3.38; normalized (MOS-1)/4.",
    "raw_value": 3.38,
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "voice": "–",
    "strong_prompting": false,
    "source_model_label": "Sesame1B"
  },
  {
    "model": "xai-grok-tts",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1270.95,
    "raw_value": 1270.95,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID xai-xai-tts; apiModelId xai-tts; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "xai-xai-tts",
    "source_model_label": "Grok TTS",
    "source_registry_request_id": "xai-tts",
    "counts": {
      "wins": 570,
      "losses": 367,
      "ties": 102,
      "voteCount": 1039
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "fish-audio-s2-1-pro",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1240.13,
    "raw_value": 1240.13,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID fish-s21-pro; apiModelId s2.1-pro; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "fish-s21-pro",
    "source_model_label": "S2.1-Pro",
    "source_registry_request_id": "s2.1-pro",
    "counts": {
      "wins": 508,
      "losses": 404,
      "ties": 137,
      "voteCount": 1049
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "inworld-tts-1-5-max",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1227.34,
    "raw_value": 1227.34,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID inworld-tts-15-max; apiModelId tts-1-5-max; pipeline vendor ID override inworld-tts-1.5-max; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "inworld-tts-15-max",
    "source_model_label": "TTS-1.5-max",
    "source_registry_request_id": "tts-1-5-max",
    "counts": {
      "wins": 469,
      "losses": 410,
      "ties": 144,
      "voteCount": 1023
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts",
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/pipeline/models.ts"
    ]
  },
  {
    "model": "elevenlabs-turbo-v2",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1216.04,
    "raw_value": 1216.04,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID elevenlabs-turbo-v2; apiModelId eleven_turbo_v2; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "elevenlabs-turbo-v2",
    "source_model_label": "Turbo v2",
    "source_registry_request_id": "eleven_turbo_v2",
    "counts": {
      "wins": 466,
      "losses": 440,
      "ties": 140,
      "voteCount": 1046
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "elevenlabs-flash-v2",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1215.5,
    "raw_value": 1215.5,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID elevenlabs-flash-v2; apiModelId eleven_flash_v2; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "elevenlabs-flash-v2",
    "source_model_label": "Flash v2",
    "source_registry_request_id": "eleven_flash_v2",
    "counts": {
      "wins": 467,
      "losses": 446,
      "ties": 130,
      "voteCount": 1043
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1198.77,
    "raw_value": 1198.77,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID cartesia-sonic-35; apiModelId sonic-3.5; registry status active. Current published BT rating; snapshot counters may be newer than rating fit. Sonic 3.5 is NOT Sonic 3.6 (cartesia-sonic-3-6 remains unrecovered).",
    "source_model_id": "cartesia-sonic-35",
    "source_model_label": "Sonic 3.5",
    "source_registry_request_id": "sonic-3.5",
    "counts": {
      "wins": 445,
      "losses": 471,
      "ties": 115,
      "voteCount": 1031
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "minimax-speech-02-turbo",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1195.96,
    "raw_value": 1195.96,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID minimax-speech-02-turbo; apiModelId speech-02-turbo; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "minimax-speech-02-turbo",
    "source_model_label": "Speech 2 Turbo",
    "source_registry_request_id": "speech-02-turbo",
    "counts": {
      "wins": 442,
      "losses": 460,
      "ties": 107,
      "voteCount": 1009
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "cartesia-sonic-3",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1185.67,
    "raw_value": 1185.67,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID cartesia-sonic-3; apiModelId sonic-3; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "cartesia-sonic-3",
    "source_model_label": "Sonic 3",
    "source_registry_request_id": "sonic-3",
    "counts": {
      "wins": 438,
      "losses": 502,
      "ties": 123,
      "voteCount": 1063
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "cartesia-sonic-2",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1185.06,
    "raw_value": 1185.06,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID cartesia-sonic-2; apiModelId sonic-2; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "cartesia-sonic-2",
    "source_model_label": "Sonic 2",
    "source_registry_request_id": "sonic-2",
    "counts": {
      "wins": 436,
      "losses": 490,
      "ties": 118,
      "voteCount": 1044
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "smallest-lightning-v3-1",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1111.5,
    "raw_value": 1111.5,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID smallestai-lightning-v31; apiModelId lightning_v3.1; registry status active. Current published BT rating; snapshot counters may be newer than rating fit. lightning_v3.1 is NOT Lightning v3.1 Pro (smallest-lightning-v3-1-pro remains unrecovered).",
    "source_model_id": "smallestai-lightning-v31",
    "source_model_label": "Lightning v3.1",
    "source_registry_request_id": "lightning_v3.1",
    "counts": {
      "wins": 323,
      "losses": 595,
      "ties": 132,
      "voteCount": 1050
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "neuphonic-neu-hq",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1102.8,
    "raw_value": 1102.8,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID neuphonic-neu-hq; apiModelId neu_hq; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "neuphonic-neu-hq",
    "source_model_label": "neu_hq",
    "source_registry_request_id": "neu_hq",
    "counts": {
      "wins": 312,
      "losses": 615,
      "ties": 106,
      "voteCount": 1033
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "gradium-tts",
    "bench": "vapi-humanness-bt-20260925",
    "score": 1089.81,
    "raw_value": 1089.81,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID gradium-gradium-tts; apiModelId gradiumtts; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "gradium-gradium-tts",
    "source_model_label": "Gradium TTS",
    "source_registry_request_id": "gradiumtts",
    "counts": {
      "wins": 305,
      "losses": 626,
      "ties": 99,
      "voteCount": 1030
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "cartesia-sonic",
    "bench": "vapi-humanness-bt-20260925",
    "score": 974.99,
    "raw_value": 974.99,
    "raw_unit": "rating points",
    "source": "https://humannessindex.vapi.ai/api/models",
    "source_id": "speech-vapi-humanness-20260925",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "API snapshot retrieval/review date; fit and synthesis dates unspecified",
    "note": "Exact registry model ID cartesia-sonic; apiModelId sonic; registry status active. Current published BT rating; snapshot counters may be newer than rating fit.",
    "source_model_id": "cartesia-sonic",
    "source_model_label": "Sonic",
    "source_registry_request_id": "sonic",
    "counts": {
      "wins": 177,
      "losses": 775,
      "ties": 73,
      "voteCount": 1025
    },
    "snapshot_total_unique_votes": 13508,
    "data_license": "CC BY 4.0",
    "identity_sources": [
      "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/catalog/models.ts"
    ]
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "ztts1-fleursr-en-ttsds2-prosody",
    "score": 0.8754,
    "raw_value": 87.54,
    "scale_divisor": 100,
    "source": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md",
    "source_id": "speech-ztts1-eval-apache",
    "by": "independent",
    "date": "2026-06-22",
    "date_basis": "commit date of pinned RESULTS_CURRENT.md; generation dates unspecified",
    "note": "RESULTS_CURRENT.md §3 'Zero-shot/en' PROSODY mean column 'fish-s2-pro' = 87.54. fishaudio/s2-pro, clone mode (run_generate_omni.sh). Components retained as provenance only.",
    "source_model_label": "fish-s2-pro",
    "identity_sources": [
      "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/run_generate_omni.sh"
    ],
    "components": {
      "MPM": 95.02,
      "Pitch": 91.49,
      "Hubert Token SR": 82.48,
      "Allosaurus SR": 81.19
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "ztts1-fleursr-en-ttsds2-prosody",
    "score": 0.8688,
    "raw_value": 86.88,
    "scale_divisor": 100,
    "source": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md",
    "source_id": "speech-ztts1-eval-apache",
    "by": "independent",
    "date": "2026-06-22",
    "date_basis": "commit date of pinned RESULTS_CURRENT.md; generation dates unspecified",
    "note": "RESULTS_CURRENT.md §3 'Zero-shot/en' PROSODY mean column 'qwen3-1.7b' = 86.88. Qwen/Qwen3-TTS-12Hz-1.7B-Base, clone mode (run_generate_omni.sh key qwen3-tts-1.7b). Components retained as provenance only.",
    "source_model_label": "qwen3-1.7b",
    "identity_sources": [
      "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/run_generate_omni.sh"
    ],
    "components": {
      "MPM": 96.84,
      "Pitch": 94.01,
      "Hubert Token SR": 75.13,
      "Allosaurus SR": 81.54
    }
  },
  {
    "model": "voxcpm2",
    "bench": "ztts1-fleursr-en-ttsds2-prosody",
    "score": 0.8858,
    "raw_value": 88.58,
    "scale_divisor": 100,
    "source": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md",
    "source_id": "speech-ztts1-eval-apache",
    "by": "independent",
    "date": "2026-06-22",
    "date_basis": "commit date of pinned RESULTS_CURRENT.md; generation dates unspecified",
    "note": "RESULTS_CURRENT.md §3 'Zero-shot/en' PROSODY mean column 'voxcpm2' = 88.58. openbmb/VoxCPM2, clone mode (run_generate_omni.sh). Components retained as provenance only.",
    "source_model_label": "voxcpm2",
    "identity_sources": [
      "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/run_generate_omni.sh"
    ],
    "components": {
      "MPM": 98.62,
      "Pitch": 95.43,
      "Hubert Token SR": 76.55,
      "Allosaurus SR": 83.71
    }
  },
  {
    "model": "zonos2-8b",
    "bench": "ztts1-fleursr-en-ttsds2-prosody",
    "score": 0.8745,
    "raw_value": 87.45,
    "scale_divisor": 100,
    "source": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md",
    "source_id": "speech-ztts1-eval-apache",
    "by": "lab",
    "date": "2026-06-22",
    "date_basis": "commit date of pinned RESULTS_CURRENT.md; generation dates unspecified",
    "note": "RESULTS_CURRENT.md §3 'Zero-shot/en' PROSODY mean column 'zonos2' = 87.45. Zyphra's released ZONOS2 (Zyphra/ZONOS2) reference eval config ('zonos2'); the separate 'zonos2-stability' stable-mode column is not imported. Components retained as provenance only.",
    "source_model_label": "zonos2",
    "identity_sources": [
      "https://github.com/Zyphra/ZONOS2"
    ],
    "components": {
      "MPM": 98.52,
      "Pitch": 78.79,
      "Hubert Token SR": 80.89,
      "Allosaurus SR": 91.6
    }
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "ztts1-fleursr-en-ttsds2-prosody",
    "score": 0.8762,
    "raw_value": 87.62,
    "scale_divisor": 100,
    "source": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md",
    "source_id": "speech-ztts1-eval-apache",
    "by": "independent",
    "date": "2026-06-22",
    "date_basis": "commit date of pinned RESULTS_CURRENT.md; generation dates unspecified",
    "note": "RESULTS_CURRENT.md §3 'Zero-shot/en' PROSODY mean column 'gemini-3.1-flash' = 87.62. API model gemini-3.1-flash-tts-preview (scripts/generate_gemini.py MODEL_IDS['3.1-flash']), prebuilt voices rotated per utterance, no cloning. Components retained as provenance only.",
    "source_model_label": "gemini-3.1-flash",
    "identity_sources": [
      "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/scripts/generate_gemini.py"
    ],
    "components": {
      "MPM": 95.84,
      "Pitch": 84.95,
      "Hubert Token SR": 84.25,
      "Allosaurus SR": 85.44
    }
  },
  {
    "model": "granite-speech-5-0-470m-turboctc",
    "bench": "granite5-report-oasr-public7-wer",
    "score": 0.0572,
    "raw_value": 5.72,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.20104v1",
    "source_id": "speech-stt-granite5-report-bysa",
    "by": "lab",
    "date": "2026-09-17",
    "date_basis": "arXiv v1 submission date; run date not stated",
    "note": "Table 3 'Robust fine tuning' (final training stage) AVG WER 5.72% over AMI 8.32, Earnings22 8.54, Gigaspeech 9.93, LS-clean 1.32, LS-other 2.54, SPGI 3.35, VoxPopuli 6.03. Model granite-speech-5.0-470m-turboctc (470M CTC, greedy decoding). Earlier-stage rows (Base 5.83, LLM distillation 5.75) not imported.",
    "per_set_wer_percent": {
      "AMI": 8.32,
      "Earnings22": 8.54,
      "Gigaspeech": 9.93,
      "LS-clean": 1.32,
      "LS-other": 2.54,
      "SPGI": 3.35,
      "VoxPopuli": 6.03
    },
    "data_license": "CC BY-SA 4.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "qwen-audio3-asr-panelb-zh4-cer",
    "score": 0.06555,
    "raw_value": 6.555,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "independent",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'GPT-4o Transcribe'. Derived equal-set mean of AISHELL-1 3.34, AISHELL-2 4.47, FLEURS-zh 6.14, WeNetSpeech-net 12.27 (%). Exact label 'GPT-4o Transcribe' (OpenAI gpt-4o-transcribe API); API snapshot not stated.",
    "per_set_percent": {
      "AISHELL-1": 3.34,
      "AISHELL-2": 4.47,
      "FLEURS-zh": 6.14,
      "WeNetSpeech-net": 12.27
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "qwen-audio3-asr-panelb-en3-wer",
    "score": 0.037467,
    "raw_value": 3.746667,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "independent",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'GPT-4o Transcribe'. Derived equal-set mean of FLEURS-en 5.79, LibriSpeech-clean 1.58, LibriSpeech-other 3.87 (%). Exact label 'GPT-4o Transcribe' (OpenAI gpt-4o-transcribe API); API snapshot not stated.",
    "per_set_percent": {
      "FLEURS-en": 5.79,
      "LibriSpeech-clean": 1.58,
      "LibriSpeech-other": 3.87
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "fun-asr-flash",
    "bench": "qwen-audio3-asr-panelb-zh4-cer",
    "score": 0.0301,
    "raw_value": 3.01,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "lab",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Fun-ASR-Flash'. Derived equal-set mean of AISHELL-1 0.76, AISHELL-2 1.93, FLEURS-zh 5.52, WeNetSpeech-net 3.83 (%). Fun-ASR-Flash API; Fun-ASR is the report authors' own predecessor model family (lab).",
    "per_set_percent": {
      "AISHELL-1": 0.76,
      "AISHELL-2": 1.93,
      "FLEURS-zh": 5.52,
      "WeNetSpeech-net": 3.83
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "fun-asr-flash",
    "bench": "qwen-audio3-asr-panelb-en3-wer",
    "score": 0.029367,
    "raw_value": 2.936667,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "lab",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Fun-ASR-Flash'. Derived equal-set mean of FLEURS-en 4.95, LibriSpeech-clean 1.35, LibriSpeech-other 2.51 (%). Fun-ASR-Flash API; Fun-ASR is the report authors' own predecessor model family (lab).",
    "per_set_percent": {
      "FLEURS-en": 4.95,
      "LibriSpeech-clean": 1.35,
      "LibriSpeech-other": 2.51
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "tencent-hy-asr-3-0-preview",
    "bench": "qwen-audio3-asr-panelb-zh4-cer",
    "score": 0.023675,
    "raw_value": 2.3675,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "independent",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Tencent Hy-ASR-3.0-preview'. Derived equal-set mean of AISHELL-1 0.78, AISHELL-2 2.15, FLEURS-zh 2.38, WeNetSpeech-net 4.16 (%). Exact label 'Tencent Hy-ASR-3.0-preview' as printed.",
    "per_set_percent": {
      "AISHELL-1": 0.78,
      "AISHELL-2": 2.15,
      "FLEURS-zh": 2.38,
      "WeNetSpeech-net": 4.16
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "tencent-hy-asr-3-0-preview",
    "bench": "qwen-audio3-asr-panelb-en3-wer",
    "score": 0.032433,
    "raw_value": 3.243333,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "independent",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Tencent Hy-ASR-3.0-preview'. Derived equal-set mean of FLEURS-en 5.45, LibriSpeech-clean 1.6, LibriSpeech-other 2.68 (%). Exact label 'Tencent Hy-ASR-3.0-preview' as printed.",
    "per_set_percent": {
      "FLEURS-en": 5.45,
      "LibriSpeech-clean": 1.6,
      "LibriSpeech-other": 2.68
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "qwen-audio-3-0-asr",
    "bench": "qwen-audio3-asr-panelb-zh4-cer",
    "score": 0.0236,
    "raw_value": 2.36,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "lab",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Qwen-Audio-3.0-ASR'. Derived equal-set mean of AISHELL-1 1.03, AISHELL-2 2.02, FLEURS-zh 2.08, WeNetSpeech-net 4.31 (%). Report's own model, non-streaming Qwen-Audio-3.0-ASR (not the -Streaming variant).",
    "per_set_percent": {
      "AISHELL-1": 1.03,
      "AISHELL-2": 2.02,
      "FLEURS-zh": 2.08,
      "WeNetSpeech-net": 4.31
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "qwen-audio-3-0-asr",
    "bench": "qwen-audio3-asr-panelb-en3-wer",
    "score": 0.025333,
    "raw_value": 2.533333,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.07549v2",
    "source_id": "speech-stt-qwen-audio3-asr-report",
    "by": "lab",
    "date": "2026-09-09",
    "date_basis": "arXiv v2 date; API run dates not stated",
    "note": "Table 1 Panel B 'Qwen-Audio-3.0-ASR'. Derived equal-set mean of FLEURS-en 4.17, LibriSpeech-clean 1.19, LibriSpeech-other 2.24 (%). Report's own model, non-streaming Qwen-Audio-3.0-ASR (not the -Streaming variant).",
    "per_set_percent": {
      "FLEURS-en": 4.17,
      "LibriSpeech-clean": 1.19,
      "LibriSpeech-other": 2.24
    },
    "derived": "unweighted mean of printed per-set percentages",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0147,
    "raw_value": 1.47,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'openai-gpt-4o-transcribe': corpus WER 1.47% (95% CI 1.33-1.63), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. OpenAI model=gpt-4o-transcribe, /v1/audio/transcriptions, response_format=json.",
    "ci95_percent": [
      1.33,
      1.63
    ],
    "sample_count": 2620,
    "engine_label": "openai-gpt-4o-transcribe",
    "data_license": "MIT"
  },
  {
    "model": "assemblyai-universal-3-5-pro",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0155,
    "raw_value": 1.55,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'assemblyai-universal': corpus WER 1.55% (95% CI 1.4-1.69), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. AssemblyAI pre-recorded upload/poll API; adapter default speech_models=['universal-3-5-pro'] (unchanged from the initial import d7c6056 to the pinned commit); README labels the row 'AssemblyAI Universal-3.5'. Async/pre-recorded, not Universal-3.5 Pro Realtime.",
    "ci95_percent": [
      1.4,
      1.69
    ],
    "sample_count": 2620,
    "engine_label": "assemblyai-universal",
    "data_license": "MIT"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0222,
    "raw_value": 2.22,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'speechmatics-enhanced': corpus WER 2.22% (95% CI 2.05-2.39), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Speechmatics batch operating point 'enhanced'.",
    "ci95_percent": [
      2.05,
      2.39
    ],
    "sample_count": 2620,
    "engine_label": "speechmatics-enhanced",
    "data_license": "MIT"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.025,
    "raw_value": 2.5,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'deepgram-nova-3': corpus WER 2.5% (95% CI 2.3-2.7), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Deepgram model=nova-3 pre-recorded API.",
    "ci95_percent": [
      2.3,
      2.7
    ],
    "sample_count": 2620,
    "engine_label": "deepgram-nova-3",
    "data_license": "MIT"
  },
  {
    "model": "assemblyai-universal-3-5-pro",
    "bench": "verli-earnings22-wer",
    "score": 0.1043,
    "raw_value": 10.43,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'assemblyai-universal': corpus WER 10.43% (95% CI 9.98-10.91), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. AssemblyAI pre-recorded upload/poll API; adapter default speech_models=['universal-3-5-pro'] (unchanged from the initial import d7c6056 to the pinned commit); README labels the row 'AssemblyAI Universal-3.5'. Async/pre-recorded, not Universal-3.5 Pro Realtime.",
    "ci95_percent": [
      9.98,
      10.91
    ],
    "sample_count": 2741,
    "engine_label": "assemblyai-universal",
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "verli-earnings22-wer",
    "score": 0.112,
    "raw_value": 11.2,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'parakeet-v3': corpus WER 11.2% (95% CI 10.71-11.74), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. On-device via parakeet-mlx, weights mlx-community/parakeet-tdt-0.6b-v3 (MLX conversion of nvidia/parakeet-tdt-0.6b-v3); not the NeMo reference runtime.",
    "ci95_percent": [
      10.71,
      11.74
    ],
    "sample_count": 2741,
    "engine_label": "parakeet-v3",
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "verli-earnings22-wer",
    "score": 0.1123,
    "raw_value": 11.23,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'parakeet-v2': corpus WER 11.23% (95% CI 10.73-11.73), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. On-device via parakeet-mlx, weights mlx-community/parakeet-tdt-0.6b-v2 (MLX conversion of nvidia/parakeet-tdt-0.6b-v2); not the NeMo reference runtime.",
    "ci95_percent": [
      10.73,
      11.73
    ],
    "sample_count": 2741,
    "engine_label": "parakeet-v2",
    "data_license": "MIT"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "verli-earnings22-wer",
    "score": 0.1228,
    "raw_value": 12.28,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'speechmatics-enhanced': corpus WER 12.28% (95% CI 11.84-12.73), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Speechmatics batch operating point 'enhanced'.",
    "ci95_percent": [
      11.84,
      12.73
    ],
    "sample_count": 2741,
    "engine_label": "speechmatics-enhanced",
    "data_license": "MIT"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "verli-earnings22-wer",
    "score": 0.1613,
    "raw_value": 16.13,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'deepgram-nova-3': corpus WER 16.13% (95% CI 15.48-16.81), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Deepgram model=nova-3 pre-recorded API.",
    "ci95_percent": [
      15.48,
      16.81
    ],
    "sample_count": 2741,
    "engine_label": "deepgram-nova-3",
    "data_license": "MIT"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "verli-earnings22-wer",
    "score": 0.1855,
    "raw_value": 18.55,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'openai-gpt-4o-transcribe': corpus WER 18.55% (95% CI 17.62-19.48), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. OpenAI model=gpt-4o-transcribe, /v1/audio/transcriptions, response_format=json.",
    "ci95_percent": [
      17.62,
      19.48
    ],
    "sample_count": 2741,
    "engine_label": "openai-gpt-4o-transcribe",
    "data_license": "MIT"
  },
  {
    "model": "assemblyai-universal-3-5-pro",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0187,
    "raw_value": 1.87,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'assemblyai-universal': corpus WER 1.87% (95% CI 1.67-2.07), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. AssemblyAI pre-recorded upload/poll API; adapter default speech_models=['universal-3-5-pro'] (unchanged from the initial import d7c6056 to the pinned commit); README labels the row 'AssemblyAI Universal-3.5'. Async/pre-recorded, not Universal-3.5 Pro Realtime.",
    "ci95_percent": [
      1.67,
      2.07
    ],
    "sample_count": 1200,
    "engine_label": "assemblyai-universal",
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0204,
    "raw_value": 2.04,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'parakeet-v2': corpus WER 2.04% (95% CI 1.84-2.22), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. On-device via parakeet-mlx, weights mlx-community/parakeet-tdt-0.6b-v2 (MLX conversion of nvidia/parakeet-tdt-0.6b-v2); not the NeMo reference runtime.",
    "ci95_percent": [
      1.84,
      2.22
    ],
    "sample_count": 1200,
    "engine_label": "parakeet-v2",
    "data_license": "MIT"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0294,
    "raw_value": 2.94,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'deepgram-nova-3': corpus WER 2.94% (95% CI 2.52-3.47), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. Deepgram model=nova-3 pre-recorded API.",
    "ci95_percent": [
      2.52,
      3.47
    ],
    "sample_count": 1200,
    "engine_label": "deepgram-nova-3",
    "data_license": "MIT"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.03,
    "raw_value": 3,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'speechmatics-enhanced': corpus WER 3.0% (95% CI 2.77-3.25), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. Speechmatics batch operating point 'enhanced'.",
    "ci95_percent": [
      2.77,
      3.25
    ],
    "sample_count": 1200,
    "engine_label": "speechmatics-enhanced",
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0384,
    "raw_value": 3.84,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'parakeet-v3': corpus WER 3.84% (95% CI 3.55-4.12), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. On-device via parakeet-mlx, weights mlx-community/parakeet-tdt-0.6b-v3 (MLX conversion of nvidia/parakeet-tdt-0.6b-v3); not the NeMo reference runtime.",
    "ci95_percent": [
      3.55,
      4.12
    ],
    "sample_count": 1200,
    "engine_label": "parakeet-v3",
    "data_license": "MIT"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0949,
    "raw_value": 9.49,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'openai-gpt-4o-transcribe': corpus WER 9.49% (95% CI 8.31-10.65), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. OpenAI model=gpt-4o-transcribe, /v1/audio/transcriptions, response_format=json.",
    "ci95_percent": [
      8.31,
      10.65
    ],
    "sample_count": 1200,
    "engine_label": "openai-gpt-4o-transcribe",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.038674,
    "raw_value": 0.038674,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'elevenlabs', model_id 'scribe_v2', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.027958
      },
      "en": {
        "metric": "wer",
        "rate": 0.029115
      },
      "es": {
        "metric": "wer",
        "rate": 0.016296
      },
      "fr": {
        "metric": "wer",
        "rate": 0.038991
      },
      "ja": {
        "metric": "cer",
        "rate": 0.022642
      },
      "ko": {
        "metric": "wer",
        "rate": 0.100757
      },
      "pt": {
        "metric": "wer",
        "rate": 0.023423
      },
      "ru": {
        "metric": "wer",
        "rate": 0.027697
      },
      "zh": {
        "metric": "cer",
        "rate": 0.061183
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.039854,
    "raw_value": 0.039854,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'openai', model_id 'gpt-4o-transcribe', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.022143
      },
      "en": {
        "metric": "wer",
        "rate": 0.021836
      },
      "es": {
        "metric": "wer",
        "rate": 0.015337
      },
      "fr": {
        "metric": "wer",
        "rate": 0.0355
      },
      "ja": {
        "metric": "cer",
        "rate": 0.029577
      },
      "ko": {
        "metric": "wer",
        "rate": 0.123148
      },
      "pt": {
        "metric": "wer",
        "rate": 0.024039
      },
      "ru": {
        "metric": "wer",
        "rate": 0.028489
      },
      "zh": {
        "metric": "cer",
        "rate": 0.058617
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "google-chirp-3",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.041778,
    "raw_value": 0.041778,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'google_stt_v2', model_id 'chirp_3', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.026392
      },
      "en": {
        "metric": "wer",
        "rate": 0.026767
      },
      "es": {
        "metric": "wer",
        "rate": 0.016296
      },
      "fr": {
        "metric": "wer",
        "rate": 0.045975
      },
      "ja": {
        "metric": "cer",
        "rate": 0.029169
      },
      "ko": {
        "metric": "wer",
        "rate": 0.114916
      },
      "pt": {
        "metric": "wer",
        "rate": 0.0263
      },
      "ru": {
        "metric": "wer",
        "rate": 0.02374
      },
      "zh": {
        "metric": "cer",
        "rate": 0.066451
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.075078,
    "raw_value": 0.075078,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'deepgram', model_id 'nova-3', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.0586
      },
      "en": {
        "metric": "wer",
        "rate": 0.047429
      },
      "es": {
        "metric": "wer",
        "rate": 0.039877
      },
      "fr": {
        "metric": "wer",
        "rate": 0.075655
      },
      "ja": {
        "metric": "cer",
        "rate": 0.058746
      },
      "ko": {
        "metric": "wer",
        "rate": 0.220942
      },
      "pt": {
        "metric": "wer",
        "rate": 0.054654
      },
      "ru": {
        "metric": "wer",
        "rate": 0.045107
      },
      "zh": {
        "metric": "cer",
        "rate": 0.074689
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "mai-transcribe-1-5",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.036238,
    "raw_value": 0.036238,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'mai_transcribe', model_id 'mai-transcribe-1.5', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.02259
      },
      "en": {
        "metric": "wer",
        "rate": 0.027471
      },
      "es": {
        "metric": "wer",
        "rate": 0.016679
      },
      "fr": {
        "metric": "wer",
        "rate": 0.034336
      },
      "ja": {
        "metric": "cer",
        "rate": 0.019072
      },
      "ko": {
        "metric": "wer",
        "rate": 0.106026
      },
      "pt": {
        "metric": "wer",
        "rate": 0.025272
      },
      "ru": {
        "metric": "wer",
        "rate": 0.022158
      },
      "zh": {
        "metric": "cer",
        "rate": 0.052539
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "koedesk-fleurs200-9lang-headline-mean",
    "score": 0.051884,
    "raw_value": 0.051884,
    "raw_unit": "error ratio (fraction)",
    "source": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json",
    "source_id": "speech-stt-koedesk-ccby",
    "by": "independent",
    "date": "2026-07-05",
    "note": "Engine 'mistral_voxtral', model_id 'voxtral-mini-2602', explicit language per call, 200 utterances x 9 languages. Derived mean of per-language headline corpus rates (ja/zh CER, others WER).",
    "per_language": {
      "de": {
        "metric": "wer",
        "rate": 0.031984
      },
      "en": {
        "metric": "wer",
        "rate": 0.033341
      },
      "es": {
        "metric": "wer",
        "rate": 0.021664
      },
      "fr": {
        "metric": "wer",
        "rate": 0.049079
      },
      "ja": {
        "metric": "cer",
        "rate": 0.041815
      },
      "ko": {
        "metric": "wer",
        "rate": 0.151136
      },
      "pt": {
        "metric": "wer",
        "rate": 0.03308
      },
      "ru": {
        "metric": "wer",
        "rate": 0.037193
      },
      "zh": {
        "metric": "cer",
        "rate": 0.067666
      }
    },
    "derived": "unweighted mean over de,en,es,fr,ja,ko,pt,ru,zh",
    "data_license": "CC BY 4.0"
  },
  {
    "model": "grok-voice-transcribe-2",
    "bench": "raven-de-read4-wer",
    "score": 0.082048,
    "raw_value": 8.204775,
    "scale_divisor": 100,
    "source": "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/BENCHMARKS.md",
    "source_id": "speech-stt-raven-mit",
    "by": "independent",
    "date": "2026-09-18",
    "note": "xAI hosted model_id 'grok-voice-transcribe-2.0' (REST /v1/stt, language=de, format=true), 400/400 utterances returned. Derived mean of per-set flozi-strict WER: avemio-german-mixed-test 6.8715%, fleurs 4.5879%, mls-de 8.3707%, voxpopuli-de 12.989%.",
    "per_set": {
      "avemio-german-mixed-test": {
        "wer_pct": 6.8715,
        "ci95": [
          5.2378,
          8.7587
        ],
        "n": 100,
        "dataset_revision": "17bcbc196567bca7eae3e94942588b5678fd748e"
      },
      "fleurs": {
        "wer_pct": 4.5879,
        "ci95": [
          3.4322,
          5.8978
        ],
        "n": 100,
        "dataset_revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd"
      },
      "mls-de": {
        "wer_pct": 8.3707,
        "ci95": [
          6.8719,
          9.9504
        ],
        "n": 100,
        "dataset_revision": "2e83e61823b4c47dcbcb1980bb88601274127609"
      },
      "voxpopuli-de": {
        "wer_pct": 12.989,
        "ci95": [
          10.8834,
          15.3167
        ],
        "n": 100,
        "dataset_revision": "42f01879c780b4a2e90ec0b4f616c2ece526e4f1"
      }
    },
    "raw_sources": [
      "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/artifacts/2026-09-18-avemio-german-mixed-test-grok-voice-transcribe-2-0/xai-grok-voice-transcribe-2.0/expected.json",
      "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/artifacts/2026-09-18-fleurs-grok-voice-transcribe-2-0/xai-grok-voice-transcribe-2.0/expected.json",
      "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/artifacts/2026-09-18-mls-de-grok-voice-transcribe-2-0/xai-grok-voice-transcribe-2.0/expected.json",
      "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/artifacts/2026-09-18-voxpopuli-de-grok-voice-transcribe-2-0/xai-grok-voice-transcribe-2.0/expected.json"
    ],
    "derived": "unweighted mean of four set-level corpus WERs",
    "data_license": "MIT"
  },
  {
    "model": "soniox-v5-async",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.0813,
    "raw_value": 0.0813,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'soniox': corpus CER 0.0813 (95% CI 0.0733-0.0897), WER 0.215 companion only, files_scored 1001/1001. Soniox model=stt-async-v5 (async API), language_hints=[bn]. Recomputed from committed per-file hypotheses with the repository normalizer: matches to 4 d.p.",
    "ci95": [
      0.0733,
      0.0897
    ],
    "sample_count": 1001,
    "companion_metric": {
      "word_error_rate": 0.215,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.101,
    "raw_value": 0.101,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'deepgram_nova3': corpus CER 0.101 (95% CI 0.0921-0.1107), WER 0.2391 companion only, files_scored 1000/1001. Deepgram model=nova-3, language=bn (1,000/1,001 clips returned). Recomputed from committed per-file hypotheses with the repository normalizer: matches to 4 d.p.",
    "ci95": [
      0.0921,
      0.1107
    ],
    "sample_count": 1000,
    "companion_metric": {
      "word_error_rate": 0.2391,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.2866,
    "raw_value": 0.2866,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'openai_4o': corpus CER 0.2866 (95% CI 0.2642-0.3107), WER 0.4507 companion only, files_scored 1001/1001. OpenAI model=gpt-4o-transcribe, language=bn, /v1 audio transcriptions. Recomputed from committed per-file hypotheses with the repository normalizer: matches to 4 d.p.",
    "ci95": [
      0.2642,
      0.3107
    ],
    "sample_count": 1001,
    "companion_metric": {
      "word_error_rate": 0.4507,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.3069,
    "raw_value": 0.3069,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'groq_whisper': corpus CER 0.3069 (95% CI 0.2915-0.3238), WER 0.7165 companion only, files_scored 1001/1001. Groq-hosted model=whisper-large-v3 (OpenAI-compatible endpoint), language=bn; provider-hosted open weights, runtime/precision not disclosed. Recomputed from committed per-file hypotheses with the repository normalizer: matches to 4 d.p.",
    "ci95": [
      0.2915,
      0.3238
    ],
    "sample_count": 1001,
    "companion_metric": {
      "word_error_rate": 0.7165,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "sarvam-saarika-v2-5",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.0685,
    "raw_value": 0.0685,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'sarvam_saarika': corpus CER 0.0685 (95% CI 0.0623-0.075), WER 0.1817 companion only, files_scored 1001/1001. Sarvam AI model=saarika:v2.5, language_code=bn-IN. Recomputed from committed per-file hypotheses with the repository normalizer: matches to 4 d.p.",
    "ci95": [
      0.0623,
      0.075
    ],
    "sample_count": 1001,
    "companion_metric": {
      "word_error_rate": 0.1817,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "perle-cs4-mean-wer",
    "score": 0.132,
    "raw_value": 13.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/pdf/2605.19069v3",
    "source_id": "speech-stt-perle-cs-report",
    "by": "independent",
    "date": "2026-05-22",
    "date_basis": "v3 date; APIs accessed May 2026",
    "note": "Table 4 'ElevenLabs Scribe v2': mean WER 13.2% across 4 code-switching pairs; companion mean BERTScore 0.936 not imported.",
    "companion_metric": {
      "bertscore_f1": 0.936,
      "ranked": false
    },
    "data_license": "CC BY 4.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "perle-cs4-mean-wer",
    "score": 0.386,
    "raw_value": 38.6,
    "scale_divisor": 100,
    "source": "https://arxiv.org/pdf/2605.19069v3",
    "source_id": "speech-stt-perle-cs-report",
    "by": "independent",
    "date": "2026-05-22",
    "date_basis": "v3 date; APIs accessed May 2026",
    "note": "Table 4 'OpenAI gpt-4o-transcribe': mean WER 38.6% across 4 code-switching pairs; companion mean BERTScore 0.856 not imported.",
    "companion_metric": {
      "bertscore_f1": 0.856,
      "ranked": false
    },
    "data_license": "CC BY 4.0"
  },
  {
    "model": "google-chirp-3",
    "bench": "perle-cs4-mean-wer",
    "score": 0.394,
    "raw_value": 39.4,
    "scale_divisor": 100,
    "source": "https://arxiv.org/pdf/2605.19069v3",
    "source_id": "speech-stt-perle-cs-report",
    "by": "independent",
    "date": "2026-05-22",
    "date_basis": "v3 date; APIs accessed May 2026",
    "note": "Table 4 'Google Chirp 3': mean WER 39.4% across 4 code-switching pairs; companion mean BERTScore 0.862 not imported.",
    "companion_metric": {
      "bertscore_f1": 0.862,
      "ranked": false
    },
    "data_license": "CC BY 4.0"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "lmarena-text",
    "score": 1501.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-fable-5.1-max'; 11,800 votes; published 95% interval 1494.7–1507.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-3",
    "bench": "lmarena-text",
    "score": 1494.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'muse-spark-1.3-max'; 12,343 votes; published 95% interval 1488.1–1500.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "lmarena-text",
    "score": 1494.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.8-flash-high'; 26,298 votes; published 95% interval 1489.7–1499.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-5",
    "bench": "lmarena-text",
    "score": 1489.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-5-max'; 28,937 votes; published 95% interval 1484.5–1494.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "lmarena-text",
    "score": 1487,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.1-pro-preview'; 121,806 votes; published 95% interval 1484.0–1490.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "lmarena-text",
    "score": 1481.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.8-max'; 23,353 votes; published 95% interval 1476.4–1486.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-astra",
    "bench": "lmarena-text",
    "score": 1477.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-6-astra-max'; 9,156 votes; published 95% interval 1470.2–1484.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "lmarena-text",
    "score": 1461.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-sonnet-5-high'; 45,739 votes; published 95% interval 1457.6–1466.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "lmarena-text",
    "score": 1455,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.5-flash-lite'; 35,763 votes; published 95% interval 1450.5–1459.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-6",
    "bench": "lmarena-text",
    "score": 1453.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.6-high'; 23,680 votes; published 95% interval 1448.7–1458.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "lmarena-text",
    "score": 1413.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-haiku-4-5-20251001'; 143,402 votes; published 95% interval 1411.3–1416.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-2-3b",
    "bench": "lmarena-text",
    "score": 1289.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-4.2-3b'; 3,247 votes; published 95% interval 1277.9–1301.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "lmarena-text",
    "score": 1317.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-oss-20b'; 10,368 votes; published 95% interval 1311.2–1324.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-2-8b",
    "bench": "lmarena-text",
    "score": 1288.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-4.2-8b'; 3,237 votes; published 95% interval 1276.9–1299.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-27b",
    "bench": "lmarena-text",
    "score": 1437.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.8-27b'; 18,462 votes; published 95% interval 1432.3–1443.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-31b",
    "bench": "lmarena-text",
    "score": 1452.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-4-31b'; 6,133 votes; published 95% interval 1445.4–1460.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "lmarena-text",
    "score": 1437.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-4-26b-a4b'; 6,081 votes; published 95% interval 1430.0–1444.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "lmarena-text",
    "score": 1364.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.7-flash'; 11,977 votes; published 95% interval 1359.1–1370.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "lmarena-text",
    "score": 1351.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-oss-120b'; 30,018 votes; published 95% interval 1347.2–1356.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "lmarena-text",
    "score": 1416.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-122b-a10b'; 29,651 votes; published 95% interval 1411.8–1420.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "lmarena-text",
    "score": 1360.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nvidia-nemotron-3-super-120b-a12b'; 7,859 votes; published 95% interval 1353.3–1367.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "lmarena-text",
    "score": 1473,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5.3-flash'; 22,971 votes; published 95% interval 1467.7–1478.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m3",
    "bench": "lmarena-text",
    "score": 1440.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m3'; 58,757 votes; published 95% interval 1436.1–1444.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-3",
    "bench": "lmarena-text",
    "score": 1478.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5.3-max'; 17,857 votes; published 95% interval 1472.9–1484.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "lmarena-text",
    "score": 1464.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v4-pro-high-20260813'; 9,780 votes; published 95% interval 1457.7–1470.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k3",
    "bench": "lmarena-text",
    "score": 1488.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k3-max'; 28,280 votes; published 95% interval 1483.6–1493.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "lmarena-text",
    "score": 1483.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.6-sol-xhigh'; 36,656 votes; published 95% interval 1479.5–1488.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "lmarena-text",
    "score": 1465.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.6-terra-xhigh'; 37,663 votes; published 95% interval 1461.1–1469.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "lmarena-text",
    "score": 1453.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.6-luna-xhigh'; 38,275 votes; published 95% interval 1448.7–1457.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5",
    "bench": "lmarena-text",
    "score": 1481.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.5-high'; 69,202 votes; published 95% interval 1477.7–1485.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4",
    "bench": "lmarena-text",
    "score": 1475.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.4-high'; 64,642 votes; published 95% interval 1471.5–1479.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "lmarena-text",
    "score": 1447.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.4-mini-high'; 63,290 votes; published 95% interval 1443.3–1450.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "lmarena-text",
    "score": 1401.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.4-nano-high'; 62,249 votes; published 95% interval 1397.5–1405.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2",
    "bench": "lmarena-text",
    "score": 1437.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.2-high'; 49,542 votes; published 95% interval 1433.7–1440.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1",
    "bench": "lmarena-text",
    "score": 1455.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.1-high'; 41,577 votes; published 95% interval 1452.1–1459.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5",
    "bench": "lmarena-text",
    "score": 1434.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5-high'; 31,489 votes; published 95% interval 1430.0–1439.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-mini",
    "bench": "lmarena-text",
    "score": 1389.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5-mini-high'; 26,638 votes; published 95% interval 1385.1–1394.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-nano",
    "bench": "lmarena-text",
    "score": 1337.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5-nano-high'; 8,112 votes; published 95% interval 1330.6–1344.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o3",
    "bench": "lmarena-text",
    "score": 1431.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o3-2025-04-16'; 58,583 votes; published 95% interval 1428.0–1435.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o4-mini",
    "bench": "lmarena-text",
    "score": 1390.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o4-mini-2025-04-16'; 44,626 votes; published 95% interval 1386.8–1394.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1",
    "bench": "lmarena-text",
    "score": 1414.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4.1-2025-04-14'; 49,991 votes; published 95% interval 1411.0–1418.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "lmarena-text",
    "score": 1382.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4.1-mini-2025-04-14'; 38,547 votes; published 95% interval 1378.4–1387.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "lmarena-text",
    "score": 1481.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-8-high'; 63,974 votes; published 95% interval 1477.6–1485.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "lmarena-text",
    "score": 1501.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-7-high'; 64,946 votes; published 95% interval 1497.5–1505.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "lmarena-text",
    "score": 1504.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-6-high'; 77,636 votes; published 95% interval 1501.3–1508.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "lmarena-text",
    "score": 1472.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-sonnet-4-6'; 70,747 votes; published 95% interval 1468.6–1475.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "lmarena-text",
    "score": 1473.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-5-20251101-high-32k'; 37,448 votes; published 95% interval 1469.8–1477.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "lmarena-text",
    "score": 1456.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-sonnet-4-5-20250929-high-32k'; 83,670 votes; published 95% interval 1453.8–1459.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "lmarena-text",
    "score": 1487.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.7-flash-high'; 21,539 votes; published 95% interval 1482.5–1492.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "lmarena-text",
    "score": 1483,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.6-flash-high'; 36,070 votes; published 95% interval 1478.6–1487.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "lmarena-text",
    "score": 1477.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.5-flash-high'; 48,420 votes; published 95% interval 1473.5–1481.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "lmarena-text",
    "score": 1432.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3.1-flash-lite-preview'; 64,047 votes; published 95% interval 1429.1–1436.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "lmarena-text",
    "score": 1445.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.5-pro'; 124,834 votes; published 95% interval 1443.1–1448.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "lmarena-text",
    "score": 1409.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.5-flash'; 125,012 votes; published 95% interval 1406.9–1411.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-lite-preview-06-17",
    "bench": "lmarena-text",
    "score": 1374.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-lite-preview-06-17-thinking'; 32,175 votes; published 95% interval 1370.3–1379.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-5",
    "bench": "lmarena-text",
    "score": 1465.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.5'; 39,789 votes; published 95% interval 1461.4–1470.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-3",
    "bench": "lmarena-text",
    "score": 1441.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.3'; 71,150 votes; published 95% interval 1437.8–1445.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20",
    "bench": "lmarena-text",
    "score": 1471.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.20-beta-0309-reasoning'; 66,334 votes; published 95% interval 1467.8–1475.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-2",
    "bench": "lmarena-text",
    "score": 1493.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'muse-spark-1.2 (xHigh)'; 4,356 votes; published 95% interval 1484.4–1502.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-1",
    "bench": "lmarena-text",
    "score": 1491.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'muse-spark-1.1'; 37,238 votes; published 95% interval 1486.9–1495.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-7-max-preview",
    "bench": "lmarena-text",
    "score": 1474.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.7-max-preview'; 3,920 votes; published 95% interval 1465.1–1484.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-7-plus",
    "bench": "lmarena-text",
    "score": 1455.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.7-plus'; 41,908 votes; published 95% interval 1451.4–1460.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-6-plus",
    "bench": "lmarena-text",
    "score": 1443.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.6-plus'; 48,062 votes; published 95% interval 1439.3–1447.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-2-lite",
    "bench": "lmarena-text",
    "score": 1335,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nova-2-lite'; 12,392 votes; published 95% interval 1328.9–1341.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-micro",
    "bench": "lmarena-text",
    "score": 1240.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-micro-v1.0'; 19,364 votes; published 95% interval 1235.5–1245.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "solar-pro-4",
    "bench": "lmarena-text",
    "score": 1386.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'solar-pro4'; 15,819 votes; published 95% interval 1380.5–1392.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "lmarena-text",
    "score": 1293.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-70b-instruct'; 55,240 votes; published 95% interval 1289.6–1297.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "lmarena-text",
    "score": 1335.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-405b-instruct-bf16'; 41,375 votes; published 95% interval 1331.6–1339.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "lmarena-text",
    "score": 1317.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.3-70b-instruct'; 54,366 votes; published 95% interval 1314.3–1321.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-4-scout",
    "bench": "lmarena-text",
    "score": 1321.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-4-scout-17b-16e-instruct'; 29,667 votes; published 95% interval 1316.9–1326.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-4-maverick",
    "bench": "lmarena-text",
    "score": 1327,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-4-maverick-17b-128e-instruct'; 39,262 votes; published 95% interval 1322.7–1331.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "lmarena-text",
    "score": 1302.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen2.5-72b-instruct'; 39,406 votes; published 95% interval 1298.8–1306.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "lmarena-text",
    "score": 1375.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-235b-a22b'; 25,825 votes; published 95% interval 1370.5–1380.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "lmarena-text",
    "score": 1422.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-235b-a22b-instruct-2507'; 97,788 votes; published 95% interval 1419.8–1425.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "lmarena-text",
    "score": 1400,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-235b-a22b-thinking-2507'; 8,774 votes; published 95% interval 1393.4–1406.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "lmarena-text",
    "score": 1387.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-coder-480b-a35b-instruct'; 25,006 votes; published 95% interval 1382.3–1392.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "lmarena-text",
    "score": 1399,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-next-80b-a3b-instruct'; 22,603 votes; published 95% interval 1394.2–1403.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "lmarena-text",
    "score": 1369.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-next-80b-a3b-thinking'; 13,371 votes; published 95% interval 1363.5–1375.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "lmarena-text",
    "score": 1441.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-397b-a17b'; 88,660 votes; published 95% interval 1438.7–1445.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3",
    "bench": "lmarena-text",
    "score": 1358.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3'; 21,770 votes; published 95% interval 1353.7–1363.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "lmarena-text",
    "score": 1395.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3-0324'; 44,758 votes; published 95% interval 1391.7–1399.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-r1",
    "bench": "lmarena-text",
    "score": 1398.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-r1'; 18,524 votes; published 95% interval 1393.2–1403.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "lmarena-text",
    "score": 1421.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-r1-0528'; 18,059 votes; published 95% interval 1415.9–1427.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-1",
    "bench": "lmarena-text",
    "score": 1415.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3.1-thinking'; 11,409 votes; published 95% interval 1409.3–1422.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "lmarena-text",
    "score": 1422.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3.2-thinking'; 41,975 votes; published 95% interval 1419.2–1426.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2",
    "bench": "lmarena-text",
    "score": 1418.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k2-0711-preview'; 26,959 votes; published 95% interval 1413.3–1423.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "lmarena-text",
    "score": 1430.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k2-thinking-turbo'; 63,210 votes; published 95% interval 1427.0–1433.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-5",
    "bench": "lmarena-text",
    "score": 1450.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k2.5-thinking'; 74,565 votes; published 95% interval 1446.8–1453.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-6",
    "bench": "lmarena-text",
    "score": 1461,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k2.6'; 39,936 votes; published 95% interval 1456.5–1465.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-5",
    "bench": "lmarena-text",
    "score": 1411.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.5'; 23,661 votes; published 95% interval 1406.3–1416.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-5-air",
    "bench": "lmarena-text",
    "score": 1373.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.5-air'; 30,454 votes; published 95% interval 1369.0–1377.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-6",
    "bench": "lmarena-text",
    "score": 1424.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.6'; 35,662 votes; published 95% interval 1420.2–1428.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-7",
    "bench": "lmarena-text",
    "score": 1441.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.7'; 12,228 votes; published 95% interval 1435.4–1447.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5",
    "bench": "lmarena-text",
    "score": 1457.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5'; 29,174 votes; published 95% interval 1453.4–1461.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-1",
    "bench": "lmarena-text",
    "score": 1464.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5.1'; 58,358 votes; published 95% interval 1460.9–1468.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-2",
    "bench": "lmarena-text",
    "score": 1475.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5.2-max'; 45,399 votes; published 95% interval 1471.5–1480.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m1",
    "bench": "lmarena-text",
    "score": 1363.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m1'; 34,042 votes; published 95% interval 1359.6–1368.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2",
    "bench": "lmarena-text",
    "score": 1343.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m2'; 6,817 votes; published 95% interval 1335.6–1351.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2-5",
    "bench": "lmarena-text",
    "score": 1390.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m2.5'; 42,730 votes; published 95% interval 1386.9–1394.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2-7",
    "bench": "lmarena-text",
    "score": 1415,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m2.7'; 79,944 votes; published 95% interval 1411.6–1418.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-2",
    "bench": "lmarena-text",
    "score": 1305.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-large-2411'; 28,073 votes; published 95% interval 1301.1–1310.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-3",
    "bench": "lmarena-text",
    "score": 1413.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-large-3'; 79,452 votes; published 95% interval 1411.0–1416.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "lmarena-text",
    "score": 1426.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-medium-3.5'; 11,757 votes; published 95% interval 1420.5–1433.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "command-a",
    "bench": "lmarena-text",
    "score": 1353.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'command-a-03-2025'; 55,467 votes; published 95% interval 1350.2–1357.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "lmarena-text",
    "score": 1347.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-nemotron-ultra-253b-v1'; 2,549 votes; published 95% interval 1335.9–1359.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1-5",
    "bench": "lmarena-text",
    "score": 1342.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nvidia-llama-3.3-nemotron-super-49b-v1.5'; 3,308 votes; published 95% interval 1332.9–1352.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "lmarena-text",
    "score": 1426.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nvidia-nemotron-3-ultra-550b-a55b-nvfp4'; 11,515 votes; published 95% interval 1419.3–1433.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hy3",
    "bench": "lmarena-text",
    "score": 1456,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hy3'; 10,476 votes; published 95% interval 1449.5–1462.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-3-5-flash",
    "bench": "lmarena-text",
    "score": 1393.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'step-3.5-flash'; 60,207 votes; published 95% interval 1389.7–1396.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-5",
    "bench": "lmarena-text",
    "score": 1433.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mimo-v2.5'; 47,266 votes; published 95% interval 1429.4–1437.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "lmarena-text",
    "score": 1467.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mimo-v2.5-pro'; 70,871 votes; published 95% interval 1463.9–1471.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "inkling",
    "bench": "lmarena-text",
    "score": 1441.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'inkling'; 35,204 votes; published 95% interval 1436.9–1445.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "inkling-small",
    "bench": "lmarena-text",
    "score": 1405.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'Inkling Small'; 27,070 votes; published 95% interval 1400.3–1410.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "lmarena-text",
    "score": 1211.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-8b-instruct'; 49,605 votes; published 95% interval 1207.1–1215.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-2-3b",
    "bench": "lmarena-text",
    "score": 1166.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.2-3b-instruct'; 7,936 votes; published 95% interval 1159.0–1174.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "lmarena-text",
    "score": 1111.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.2-1b-instruct'; 8,045 votes; published 95% interval 1103.2–1119.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "lmarena-text",
    "score": 1425,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'muse-glimmer'; 3,893 votes; published 95% interval 1415.3–1434.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "lmarena-text",
    "score": 1356.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-small-2506'; 17,348 votes; published 95% interval 1351.3–1361.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "lmarena-text",
    "score": 1270.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen2.5-coder-32b-instruct'; 5,432 votes; published 95% interval 1262.5–1278.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-32b",
    "bench": "lmarena-text",
    "score": 1346.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-32b'; 3,926 votes; published 95% interval 1337.5–1356.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "lmarena-text",
    "score": 1382.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-30b-a3b-instruct-2507'; 23,174 votes; published 95% interval 1377.7–1387.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lmarena-text",
    "score": 1408.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-27b'; 28,612 votes; published 95% interval 1404.4–1413.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "lmarena-text",
    "score": 1393.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-35b-a3b'; 30,408 votes; published 95% interval 1389.6–1398.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-3-4b",
    "bench": "lmarena-text",
    "score": 1303.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-3-4b-it'; 4,171 votes; published 95% interval 1293.9–1312.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-3-12b",
    "bench": "lmarena-text",
    "score": 1341.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-3-12b-it'; 3,829 votes; published 95% interval 1332.2–1351.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-3-27b",
    "bench": "lmarena-text",
    "score": 1365.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-3-27b-it'; 46,425 votes; published 95% interval 1361.7–1369.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "lmarena-text",
    "score": 1317.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-3n-e4b-it'; 22,042 votes; published 95% interval 1312.2–1322.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-4",
    "bench": "lmarena-text",
    "score": 1256.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-4'; 24,126 votes; published 95% interval 1251.7–1260.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "lmarena-text",
    "score": 1286.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'olmo-3.1-32b-think'; 8,278 votes; published 95% interval 1279.2–1293.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "lmarena-text",
    "score": 1287,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ibm-granite-h-small'; 5,576 votes; published 95% interval 1278.5–1295.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-2-30b",
    "bench": "lmarena-text",
    "score": 1339.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-4.2-30b'; 3,401 votes; published 95% interval 1329.1–1350.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "lmarena-text",
    "score": 1313.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nvidia-nemotron-3-nano-30b-a3b-bf16'; 15,722 votes; published 95% interval 1308.0–1319.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nemotron-3-5-lightning",
    "bench": "lmarena-text",
    "score": 1349.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4'; 14,877 votes; published 95% interval 1343.5–1355.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-image-2-5-sunburst",
    "bench": "lmarena-image",
    "score": 1424.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-2.5-sunburst'; 17,752 votes; published 95% interval 1417.7–1431.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1423.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-2-5-flare",
    "bench": "lmarena-image",
    "score": 1397.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-2.5-flare'; 16,889 votes; published 95% interval 1390.9–1404.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1400.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-2",
    "bench": "lmarena-image",
    "score": 1383,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-2 (medium)'; 88,744 votes; published 95% interval 1378.9–1387.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-image-1-5",
    "bench": "lmarena-image",
    "score": 1237.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-1.5-high-fidelity'; 169,927 votes; published 95% interval 1234.3–1240.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1238.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-1",
    "bench": "lmarena-image",
    "score": 1115.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-1'; 268,453 votes; published 95% interval 1112.5–1118.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1115.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-1-mini",
    "bench": "lmarena-image",
    "score": 1109.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'gpt-image-1-mini'; 171,058 votes; published 95% interval 1106.6–1112.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-imagine-image-2",
    "bench": "lmarena-image",
    "score": 1300.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'grok-imagine-image-2.0 (low)'; 7,257 votes; published 95% interval 1293.0–1308.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-2-6",
    "bench": "lmarena-image",
    "score": 1332.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'mai-image-2.6'; 25,091 votes; published 95% interval 1326.7–1337.5; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1335), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-2",
    "bench": "lmarena-image",
    "score": 1261.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-3.1-flash-image (nano-banana-2) [web-search]'; 60,515 votes; published 95% interval 1257.6–1265.9; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1261.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-pro",
    "bench": "lmarena-image",
    "score": 1234.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-3-pro-image-preview (nano-banana-pro)'; 86,451 votes; published 95% interval 1229.2–1238.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nano-banana",
    "bench": "lmarena-image",
    "score": 1150.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-2.5-flash-image-preview (nano-banana)'; 890,539 votes; published 95% interval 1147.9–1152.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1150.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "muse-image",
    "bench": "lmarena-image",
    "score": 1274,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'muse-image'; 42,262 votes; published 95% interval 1268.9–1279.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1276), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-5-0-pro",
    "bench": "lmarena-image",
    "score": 1255.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'seedream-5.0-pro'; 94,181 votes; published 95% interval 1251.6–1259.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1256.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-4-0",
    "bench": "lmarena-image",
    "score": 1139.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'seedream-4-2k'; 12,562 votes; published 95% interval 1133.3–1146.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1140), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-image-3-0-pro",
    "bench": "lmarena-image",
    "score": 1255.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'qwen-image-3.0-pro'; 11,440 votes; published 95% interval 1248.7–1261.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1255.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-max",
    "bench": "lmarena-image",
    "score": 1162.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-max'; 123,187 votes; published 95% interval 1158.6–1165.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-pro",
    "bench": "lmarena-image",
    "score": 1153.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-pro'; 212,359 votes; published 95% interval 1150.5–1156.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1153.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "imagen-4-ultra",
    "bench": "lmarena-image",
    "score": 1147.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'imagen-ultra-4.0-generate-001'; 388,772 votes; published 95% interval 1143.4–1152.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "recraft-v4",
    "bench": "lmarena-image",
    "score": 1114.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'recraft-v4'; 137,017 votes; published 95% interval 1111.7–1118.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1115), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "wan-2-6-t2i",
    "bench": "lmarena-image",
    "score": 1137.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'wan2.6-t2i'; 233,292 votes; published 95% interval 1134.7–1140.3; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1136.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "ideogram-3",
    "bench": "lmarena-image",
    "score": 1048.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'ideogram-v3-quality'; 118,542 votes; published 95% interval 1044.7–1052.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-dev",
    "bench": "lmarena-image",
    "score": 1145.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-dev'; 77,988 votes; published 95% interval 1141.6–1149.9; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1145.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "lmarena-image",
    "score": 1070.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-klein-9b'; 150,782 votes; published 95% interval 1067.1–1073.5; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1070.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-klein-4b",
    "bench": "lmarena-image",
    "score": 1030,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-klein-4b'; 152,358 votes; published 95% interval 1026.8–1033.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-dev",
    "bench": "lmarena-image",
    "score": 969.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-1-dev-fp8'; 49,221 votes; published 95% interval 965.7–973.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-image",
    "bench": "lmarena-image",
    "score": 1056.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'qwen-image'; 84,555 votes; published 95% interval 1054.2–1059.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-image-2512",
    "bench": "lmarena-image",
    "score": 1125.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'qwen-image-2512'; 98,363 votes; published 95% interval 1121.9–1128.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "z-image-turbo",
    "bench": "lmarena-image",
    "score": 1083.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'z-image-turbo'; 24,808 votes; published 95% interval 1078.2–1088.8; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1083.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "lmarena-image",
    "score": 1149.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'hunyuan-image-3.0'; 180,550 votes; published 95% interval 1146.2–1152.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sd-3-5-large",
    "bench": "lmarena-image",
    "score": 938.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'stable-diffusion-v35-large'; 23,396 votes; published 95% interval 933.8–942.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hidream-o1-image",
    "bench": "lmarena-image",
    "score": 1116.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'hidream-o1-image'; 66,906 votes; published 95% interval 1112.5–1119.9; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1115.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "ideogram-4",
    "bench": "lmarena-image",
    "score": 1205.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'ideogram-4.0-quality'; 60,591 votes; published 95% interval 1201.1–1209.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1204.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "cosmos3-super-text2image",
    "bench": "lmarena-image",
    "score": 1151.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'Cosmos3-Super-Text2Image'; 18,485 votes; published 95% interval 1145.7–1157.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1151.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "glm-image",
    "bench": "lmarena-image",
    "score": 1012.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'glm-image'; 4,854 votes; published 95% interval 1003.2–1021.0; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1012), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gemini-omni-flash",
    "bench": "lmarena-video",
    "score": 1512.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'gemini-omni-flash'; 26,576 votes; published 95% interval 1503.9–1521.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-3-0",
    "bench": "lmarena-video",
    "score": 1476.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'wan3.0'; 2,598 votes; published 95% interval 1463.6–1488.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-0",
    "bench": "lmarena-video",
    "score": 1479.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'dreamina-seedance-2.0-720p'; 56,318 votes; published 95% interval 1471.2–1487.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "happyhorse-1-0",
    "bench": "lmarena-video",
    "score": 1427.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'happyhorse-1.0'; 22,264 votes; published 95% interval 1414.7–1439.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-7",
    "bench": "lmarena-video",
    "score": 1337.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'wan2.7-t2v'; 27,626 votes; published 95% interval 1329.9–1344.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-2-6-pro",
    "bench": "lmarena-video",
    "score": 1216.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kling-2.6-pro'; 73,362 votes; published 95% interval 1209.6–1222.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-1",
    "bench": "lmarena-video",
    "score": 1364.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'veo-3.1-audio'; 13,741 votes; published 95% interval 1349.8–1378.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-1-fast",
    "bench": "lmarena-video",
    "score": 1361.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'veo-3.1-fast-audio'; 39,484 votes; published 95% interval 1351.5–1372.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3",
    "bench": "lmarena-video",
    "score": 1340.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'veo-3-audio'; 19,002 votes; published 95% interval 1327.6–1352.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sora-2",
    "bench": "lmarena-video",
    "score": 1342.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'sora-2'; 71,947 votes; published 95% interval 1336.5–1348.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sora-2-pro",
    "bench": "lmarena-video",
    "score": 1367.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'sora-2-pro'; 54,167 votes; published 95% interval 1361.0–1374.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-imagine-video",
    "bench": "lmarena-video",
    "score": 1342.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'grok-imagine-video-720p'; 165,462 votes; published 95% interval 1335.7–1348.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "runway-gen-4-5",
    "bench": "lmarena-video",
    "score": 1224.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'runway-gen-4.5'; 48,127 votes; published 95% interval 1216.2–1233.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hailuo-2-3",
    "bench": "lmarena-video",
    "score": 1206,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'hailuo-2.3'; 89,138 votes; published 95% interval 1200.0–1211.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pixverse-v5-6",
    "bench": "lmarena-video",
    "score": 1239.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'pixverse-v5.6'; 32,434 votes; published 95% interval 1229.1–1250.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "luma-ray-3",
    "bench": "lmarena-video",
    "score": 1206.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'ray-3'; 1,136 votes; published 95% interval 1183.8–1228.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "lmarena-video",
    "score": 1256,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'seedance-v1.5-pro'; 76,171 votes; published 95% interval 1249.3–1262.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pika-2-2",
    "bench": "lmarena-video",
    "score": 1009,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'pika-v2.2'; 5,763 votes; published 95% interval 993.7–1024.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-h3",
    "bench": "lmarena-video",
    "score": 1460.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'minimax-h3'; 11,417 votes; published 95% interval 1451.7–1468.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ltx-2",
    "bench": "lmarena-video",
    "score": 1154.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'ltx-2-19b'; 75,596 votes; published 95% interval 1146.4–1161.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "lmarena-video",
    "score": 1131.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'wan-v2.2-a14b'; 10,458 votes; published 95% interval 1116.4–1146.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "lmarena-video",
    "score": 1168.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'hunyuan-video-1.5'; 4,292 votes; published 95% interval 1152.5–1185.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mochi-1",
    "bench": "lmarena-video",
    "score": 1006.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'mochi-v1'; 5,893 votes; published 95% interval 990.0–1023.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-h3",
    "bench": "lmarena-video-i2v",
    "score": 1495.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'minimax-h3'; 57,112 votes; published 95% interval 1490.2–1500.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-3-0",
    "bench": "lmarena-video-i2v",
    "score": 1479.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'wan3.0'; 3,442 votes; published 95% interval 1469.0–1490.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-0",
    "bench": "lmarena-video-i2v",
    "score": 1474.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'dreamina-seedance-2.0-720p'; 121,427 votes; published 95% interval 1467.5–1481.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-omni-flash",
    "bench": "lmarena-video-i2v",
    "score": 1464.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'gemini-omni-flash'; 87,155 votes; published 95% interval 1459.3–1470.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "happyhorse-1-0",
    "bench": "lmarena-video-i2v",
    "score": 1441.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'happyhorse-1.0'; 70,640 votes; published 95% interval 1431.8–1451.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-7",
    "bench": "lmarena-video-i2v",
    "score": 1428.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'wan2.7-i2v'; 85,521 votes; published 95% interval 1423.4–1433.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-imagine-video",
    "bench": "lmarena-video-i2v",
    "score": 1415.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'grok-imagine-video-720p'; 554,050 votes; published 95% interval 1411.2–1419.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-1",
    "bench": "lmarena-video-i2v",
    "score": 1398.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'veo-3.1-audio'; 25,116 votes; published 95% interval 1387.6–1408.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-1-fast",
    "bench": "lmarena-video-i2v",
    "score": 1385.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'veo-3.1-fast-audio'; 99,804 votes; published 95% interval 1376.2–1393.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "vidu-q3-pro",
    "bench": "lmarena-video-i2v",
    "score": 1363.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'vidu-q3-pro'; 36,679 votes; published 95% interval 1354.4–1371.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-3-0-pro",
    "bench": "lmarena-video-i2v",
    "score": 1354,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'kling-v3-pro'; 206,221 votes; published 95% interval 1348.5–1359.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3",
    "bench": "lmarena-video-i2v",
    "score": 1331.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'veo-3-audio'; 32,367 votes; published 95% interval 1319.9–1343.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "lmarena-video-i2v",
    "score": 1307.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'seedance-v1.5-pro'; 245,206 votes; published 95% interval 1300.1–1314.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pixverse-v5-6",
    "bench": "lmarena-video-i2v",
    "score": 1299.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'pixverse-v5.6'; 126,710 votes; published 95% interval 1292.0–1306.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-2-6-pro",
    "bench": "lmarena-video-i2v",
    "score": 1294,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'kling-2.6-pro'; 202,113 votes; published 95% interval 1286.2–1301.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hailuo-2-3",
    "bench": "lmarena-video-i2v",
    "score": 1262,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'hailuo-2.3'; 405,786 votes; published 95% interval 1257.1–1266.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "luma-ray-3",
    "bench": "lmarena-video-i2v",
    "score": 1225.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'ray-3'; 1,585 votes; published 95% interval 1206.9–1244.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "lmarena-video-i2v",
    "score": 1197.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'hunyuan-video-1.5'; 5,462 votes; published 95% interval 1182.1–1213.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "lmarena-video-i2v",
    "score": 1170.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'wan-v2.2-a14b'; 27,054 votes; published 95% interval 1160.2–1180.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ltx-2",
    "bench": "lmarena-video-i2v",
    "score": 1159.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'ltx-2-19b'; 262,336 votes; published 95% interval 1154.5–1164.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pika-2-2",
    "bench": "lmarena-video-i2v",
    "score": 997.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'pika-v2.2'; 8,635 votes; published 95% interval 983.4–1010.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-5-5",
    "bench": "lmarena-webdev",
    "score": 1815.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-5.5-max'; 2,062 votes; published 95% interval 1799.3–1831.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-astra",
    "bench": "lmarena-webdev",
    "score": 1787.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-6-astra-max'; 6,123 votes; published 95% interval 1777.4–1798.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "lmarena-webdev",
    "score": 1749.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-fable-5.1-max'; 6,318 votes; published 95% interval 1739.7–1758.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-5",
    "bench": "lmarena-webdev",
    "score": 1695,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-5-max'; 16,954 votes; published 95% interval 1688.5–1701.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-sol",
    "bench": "lmarena-webdev",
    "score": 1689,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-6-sol-max'; 3,411 votes; published 95% interval 1677.5–1700.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "lmarena-webdev",
    "score": 1671.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.8-max'; 3,454 votes; published 95% interval 1659.1–1683.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k3",
    "bench": "lmarena-webdev",
    "score": 1657.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'kimi-k3-max'; 16,513 votes; published 95% interval 1651.4–1664.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-3",
    "bench": "lmarena-webdev",
    "score": 1656.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'muse-spark-1.3-max'; 7,249 votes; published 95% interval 1648.0–1665.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "lmarena-webdev",
    "score": 1637.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.8-flash-next'; 6,193 votes; published 95% interval 1628.7–1646.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-7",
    "bench": "lmarena-webdev",
    "score": 1637.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.7-xhigh'; 3,216 votes; published 95% interval 1625.8–1649.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5",
    "bench": "lmarena-webdev",
    "score": 1625.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-fable-5-high'; 14,126 votes; published 95% interval 1618.9–1631.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-6",
    "bench": "lmarena-webdev",
    "score": 1619.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.6-high'; 8,315 votes; published 95% interval 1611.7–1627.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-3",
    "bench": "lmarena-webdev",
    "score": 1623,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5.3-max'; 7,658 votes; published 95% interval 1614.8–1631.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "lmarena-webdev",
    "score": 1619.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v4.1-flash-max'; 3,960 votes; published 95% interval 1609.5–1630.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "lmarena-webdev",
    "score": 1619.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.6-sol-xhigh (codex-harness)'; 17,128 votes; published 95% interval 1613.8–1625.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "lmarena-webdev",
    "score": 1615.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5.3-flash'; 10,355 votes; published 95% interval 1608.0–1623.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-2",
    "bench": "lmarena-webdev",
    "score": 1605,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5.2-max'; 14,729 votes; published 95% interval 1598.7–1611.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "lmarena-webdev",
    "score": 1591.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.7-flash-high'; 9,624 votes; published 95% interval 1584.4–1599.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-luna",
    "bench": "lmarena-webdev",
    "score": 1578.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-6-luna-max'; 6,429 votes; published 95% interval 1570.2–1587.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-27b",
    "bench": "lmarena-webdev",
    "score": 1591.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.8-27b'; 12,388 votes; published 95% interval 1584.2–1598.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "lmarena-webdev",
    "score": 1582.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.8-flash-high'; 8,524 votes; published 95% interval 1574.8–1590.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "lmarena-webdev",
    "score": 1582.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v4-pro-high-20260813'; 4,882 votes; published 95% interval 1573.3–1592.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "lmarena-webdev",
    "score": 1557.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-4-7-high'; 17,982 votes; published 95% interval 1551.6–1563.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "lmarena-webdev",
    "score": 1555.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-4-8-high'; 18,908 votes; published 95% interval 1549.8–1561.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-5",
    "bench": "lmarena-webdev",
    "score": 1551.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.5'; 11,393 votes; published 95% interval 1545.2–1558.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "lmarena-webdev",
    "score": 1545.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-4-6-high'; 19,519 votes; published 95% interval 1540.5–1551.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-1",
    "bench": "lmarena-webdev",
    "score": 1541.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'muse-spark-1.1'; 8,070 votes; published 95% interval 1534.1–1549.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "lmarena-webdev",
    "score": 1539.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-sonnet-5-high'; 14,130 votes; published 95% interval 1533.2–1545.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "lmarena-webdev",
    "score": 1536.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.6-flash-high'; 8,823 votes; published 95% interval 1528.8–1543.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-2",
    "bench": "lmarena-webdev",
    "score": 1531.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'muse-spark-1.2 (xHigh)'; 2,183 votes; published 95% interval 1518.2–1545.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "lmarena-webdev",
    "score": 1521.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-sonnet-4-6'; 22,960 votes; published 95% interval 1516.3–1526.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "lmarena-webdev",
    "score": 1518,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.6-luna-xhigh (codex-harness)'; 12,776 votes; published 95% interval 1511.6–1524.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "lmarena-webdev",
    "score": 1519.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.6-terra-xhigh (codex-harness)'; 12,526 votes; published 95% interval 1512.5–1525.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5",
    "bench": "lmarena-webdev",
    "score": 1512.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.5-xhigh (codex-harness)'; 15,873 votes; published 95% interval 1506.4–1518.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hy3",
    "bench": "lmarena-webdev",
    "score": 1507.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'hy3'; 5,802 votes; published 95% interval 1498.8–1516.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-6",
    "bench": "lmarena-webdev",
    "score": 1508.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'kimi-k2.6'; 10,208 votes; published 95% interval 1501.5–1516.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-1",
    "bench": "lmarena-webdev",
    "score": 1508.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5.1'; 10,797 votes; published 95% interval 1501.7–1515.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "lmarena-webdev",
    "score": 1499,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.5-flash-high'; 9,497 votes; published 95% interval 1491.8–1506.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "lmarena-webdev",
    "score": 1493.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-4-5-20251101-high-32k'; 11,034 votes; published 95% interval 1485.0–1501.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m3",
    "bench": "lmarena-webdev",
    "score": 1482,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'minimax-m3'; 17,613 votes; published 95% interval 1476.4–1487.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "lmarena-webdev",
    "score": 1477.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mimo-v2.5-pro'; 20,313 votes; published 95% interval 1472.3–1483.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4",
    "bench": "lmarena-webdev",
    "score": 1464.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.4-high (codex-harness)'; 1,351 votes; published 95% interval 1446.1–1483.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-6-plus",
    "bench": "lmarena-webdev",
    "score": 1460.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.6-plus'; 18,688 votes; published 95% interval 1455.2–1466.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "lmarena-webdev",
    "score": 1439.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.5-flash-lite'; 242 votes; published 95% interval 1399.2–1480.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "lmarena-webdev",
    "score": 1446.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.1-pro-preview'; 24,347 votes; published 95% interval 1441.2–1451.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-5",
    "bench": "lmarena-webdev",
    "score": 1437.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mimo-v2.5'; 15,018 votes; published 95% interval 1431.6–1443.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-5",
    "bench": "lmarena-webdev",
    "score": 1435.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'kimi-k2.5-thinking'; 20,149 votes; published 95% interval 1430.6–1441.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5",
    "bench": "lmarena-webdev",
    "score": 1434,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5'; 7,504 votes; published 95% interval 1425.9–1442.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-7",
    "bench": "lmarena-webdev",
    "score": 1434.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-4.7'; 4,007 votes; published 95% interval 1422.5–1446.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5",
    "bench": "lmarena-webdev",
    "score": 1417.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5-medium'; 3,091 votes; published 95% interval 1401.5–1432.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2",
    "bench": "lmarena-webdev",
    "score": 1415.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.2'; 1,082 votes; published 95% interval 1393.0–1437.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "inkling",
    "bench": "lmarena-webdev",
    "score": 1412.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'inkling'; 13,705 votes; published 95% interval 1406.2–1419.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "inkling-small",
    "bench": "lmarena-webdev",
    "score": 1408.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'Inkling Small'; 5,421 votes; published 95% interval 1399.2–1417.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "lmarena-webdev",
    "score": 1399.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.5-397b-a17b'; 21,280 votes; published 95% interval 1394.3–1404.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2-7",
    "bench": "lmarena-webdev",
    "score": 1397,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'minimax-m2.7'; 16,074 votes; published 95% interval 1391.2–1402.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "lmarena-webdev",
    "score": 1397,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.4-mini-high'; 11,998 votes; published 95% interval 1390.5–1403.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "lmarena-webdev",
    "score": 1393,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-sonnet-4-5-20250929-high-32k'; 13,253 votes; published 95% interval 1385.4–1400.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1",
    "bench": "lmarena-webdev",
    "score": 1394.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.1-medium'; 4,911 votes; published 95% interval 1383.3–1406.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2-5",
    "bench": "lmarena-webdev",
    "score": 1386.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'minimax-m2.5'; 7,637 votes; published 95% interval 1378.0–1394.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20",
    "bench": "lmarena-webdev",
    "score": 1374.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.20-beta-0309-reasoning'; 15,503 votes; published 95% interval 1368.0–1380.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "solar-pro-4",
    "bench": "lmarena-webdev",
    "score": 1370.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'solar-pro4'; 4,945 votes; published 95% interval 1360.0–1380.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-31b",
    "bench": "lmarena-webdev",
    "score": 1364.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemma-4-31b'; 13,325 votes; published 95% interval 1358.3–1371.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "lmarena-webdev",
    "score": 1358.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemma-4-26b-a4b'; 1,502 votes; published 95% interval 1341.6–1375.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "lmarena-webdev",
    "score": 1361.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v3.2-thinking'; 6,921 votes; published 95% interval 1352.6–1370.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "lmarena-webdev",
    "score": 1354.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'muse-glimmer'; 1,663 votes; published 95% interval 1339.3–1370.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "lmarena-webdev",
    "score": 1359.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.5-122b-a10b'; 8,249 votes; published 95% interval 1352.2–1367.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lmarena-webdev",
    "score": 1357.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.5-27b'; 7,794 votes; published 95% interval 1349.3–1365.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-3",
    "bench": "lmarena-webdev",
    "score": 1356.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.3'; 15,057 votes; published 95% interval 1350.4–1362.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-6",
    "bench": "lmarena-webdev",
    "score": 1339.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-4.6'; 6,780 votes; published 95% interval 1328.6–1350.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "lmarena-webdev",
    "score": 1329.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-haiku-4-5-20251001'; 29,184 votes; published 95% interval 1324.5–1334.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "lmarena-webdev",
    "score": 1321.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'kimi-k2-thinking-turbo'; 13,093 votes; published 95% interval 1314.0–1328.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m2",
    "bench": "lmarena-webdev",
    "score": 1296.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'minimax-m2'; 6,714 votes; published 95% interval 1285.7–1307.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "lmarena-webdev",
    "score": 1274.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3-coder-480b-a35b-instruct'; 12,719 votes; published 95% interval 1266.4–1281.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "lmarena-webdev",
    "score": 1262.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mistral-medium-3.5'; 2,322 votes; published 95% interval 1248.2–1277.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "lmarena-webdev",
    "score": 1255.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3.1-flash-lite-preview'; 14,417 votes; published 95% interval 1248.9–1262.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "lmarena-webdev",
    "score": 1253.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.5-35b-a3b'; 1,617 votes; published 95% interval 1236.0–1270.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-3",
    "bench": "lmarena-webdev",
    "score": 1229.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mistral-large-3'; 840 votes; published 95% interval 1204.3–1255.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "lmarena-webdev",
    "score": 1227,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-2.5-pro'; 2,729 votes; published 95% interval 1211.2–1242.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-7-max",
    "bench": "lmarena-webdev",
    "score": 1514.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.7-max-20260517'; 8,922 votes; published 95% interval 1507.6–1522.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "lmarena-vision",
    "score": 1301.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.8-max'; 10,040 votes; published 95% interval 1293.7–1308.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "lmarena-vision",
    "score": 1297.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-opus-4-7-high'; 22,755 votes; published 95% interval 1291.1–1304.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "lmarena-vision",
    "score": 1299.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-opus-4-6-high'; 22,328 votes; published 95% interval 1292.9–1305.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-3",
    "bench": "lmarena-vision",
    "score": 1290.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'muse-spark-1.3-max'; 3,843 votes; published 95% interval 1279.7–1300.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-2",
    "bench": "lmarena-vision",
    "score": 1292.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'muse-spark-1.2 (xHigh)'; 1,951 votes; published 95% interval 1278.5–1307.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-5",
    "bench": "lmarena-vision",
    "score": 1289,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-opus-5-high'; 15,900 votes; published 95% interval 1282.1–1295.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "lmarena-vision",
    "score": 1287.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-fable-5.1-max'; 4,312 votes; published 95% interval 1277.7–1297.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5",
    "bench": "lmarena-vision",
    "score": 1281,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.5-high'; 25,488 votes; published 95% interval 1275.0–1287.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "lmarena-vision",
    "score": 1284.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.6-sol-xhigh'; 10,421 votes; published 95% interval 1277.1–1292.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4",
    "bench": "lmarena-vision",
    "score": 1283.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.4-high'; 29,177 votes; published 95% interval 1277.9–1289.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-astra",
    "bench": "lmarena-vision",
    "score": 1284.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-6-astra-max'; 3,175 votes; published 95% interval 1273.0–1295.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "lmarena-vision",
    "score": 1284.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3.5-flash-high'; 12,336 votes; published 95% interval 1277.0–1291.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "lmarena-vision",
    "score": 1286.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-opus-4-8-high'; 18,350 votes; published 95% interval 1279.8–1292.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "lmarena-vision",
    "score": 1279.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3.6-flash-high'; 7,392 votes; published 95% interval 1271.2–1288.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-1",
    "bench": "lmarena-vision",
    "score": 1281.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'muse-spark-1.1'; 11,255 votes; published 95% interval 1273.8–1288.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-5",
    "bench": "lmarena-vision",
    "score": 1279,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4.5'; 11,039 votes; published 95% interval 1271.6–1286.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "lmarena-vision",
    "score": 1279.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3.1-pro-preview'; 45,558 votes; published 95% interval 1274.1–1284.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "lmarena-vision",
    "score": 1275.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-sonnet-4-6'; 27,429 votes; published 95% interval 1269.0–1281.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "lmarena-vision",
    "score": 1278.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'glm-5.3-flash'; 5,479 votes; published 95% interval 1269.3–1287.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "lmarena-vision",
    "score": 1268.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.6-terra-xhigh'; 10,292 votes; published 95% interval 1260.6–1275.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "lmarena-vision",
    "score": 1264.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-sonnet-5-high'; 13,029 votes; published 95% interval 1257.5–1271.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "lmarena-vision",
    "score": 1264.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3.5-flash-lite'; 7,579 votes; published 95% interval 1256.0–1272.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-6",
    "bench": "lmarena-vision",
    "score": 1263.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4.6-high'; 5,736 votes; published 95% interval 1254.4–1272.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-7-plus",
    "bench": "lmarena-vision",
    "score": 1265.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.7-plus'; 13,065 votes; published 95% interval 1258.6–1272.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-6",
    "bench": "lmarena-vision",
    "score": 1264.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'kimi-k2.6'; 16,387 votes; published 95% interval 1258.1–1271.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-31b",
    "bench": "lmarena-vision",
    "score": 1262.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemma-4-31b'; 39,111 votes; published 95% interval 1256.4–1268.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "lmarena-vision",
    "score": 1259.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.6-luna-xhigh'; 10,446 votes; published 95% interval 1252.2–1267.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20",
    "bench": "lmarena-vision",
    "score": 1255.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4.20-beta-0309-reasoning'; 26,832 votes; published 95% interval 1249.3–1261.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "lmarena-vision",
    "score": 1250.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.4-mini-high'; 25,734 votes; published 95% interval 1244.5–1257.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1",
    "bench": "lmarena-vision",
    "score": 1251.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.1-high'; 9,842 votes; published 95% interval 1243.6–1259.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-5",
    "bench": "lmarena-vision",
    "score": 1252.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'kimi-k2.5-thinking'; 28,326 votes; published 95% interval 1246.5–1257.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "lmarena-vision",
    "score": 1246.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.5-397b-a17b'; 31,441 votes; published 95% interval 1241.5–1252.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "lmarena-vision",
    "score": 1247.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.5-pro'; 88,813 votes; published 95% interval 1242.9–1252.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-27b",
    "bench": "lmarena-vision",
    "score": 1242.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.8-27b'; 6,422 votes; published 95% interval 1234.1–1251.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2",
    "bench": "lmarena-vision",
    "score": 1242.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.2-high'; 19,039 votes; published 95% interval 1236.7–1248.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-3",
    "bench": "lmarena-vision",
    "score": 1242.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4.3'; 26,300 votes; published 95% interval 1236.9–1248.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "lmarena-vision",
    "score": 1241.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemma-4-26b-a4b'; 23,954 votes; published 95% interval 1235.4–1248.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m3",
    "bench": "lmarena-vision",
    "score": 1236.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'minimax-m3'; 18,308 votes; published 95% interval 1230.0–1243.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "lmarena-vision",
    "score": 1234.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3.1-flash-lite-preview'; 27,702 votes; published 95% interval 1228.8–1240.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-5",
    "bench": "lmarena-vision",
    "score": 1233.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mimo-v2.5'; 28,160 votes; published 95% interval 1227.2–1239.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "lmarena-vision",
    "score": 1226.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.5-122b-a10b'; 13,242 votes; published 95% interval 1219.6–1233.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lmarena-vision",
    "score": 1216.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3.5-27b'; 28,993 votes; published 95% interval 1210.7–1222.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o3",
    "bench": "lmarena-vision",
    "score": 1215.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'o3-2025-04-16'; 45,593 votes; published 95% interval 1209.1–1222.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "lmarena-vision",
    "score": 1215.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.5-flash'; 60,113 votes; published 95% interval 1210.5–1219.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1",
    "bench": "lmarena-vision",
    "score": 1214,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4.1-2025-04-14'; 41,020 votes; published 95% interval 1207.3–1220.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5",
    "bench": "lmarena-vision",
    "score": 1210.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5-high'; 34,456 votes; published 95% interval 1203.1–1217.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "inkling-small",
    "bench": "lmarena-vision",
    "score": 1206.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'Inkling Small'; 6,904 votes; published 95% interval 1197.7–1215.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "lmarena-vision",
    "score": 1199.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.4-nano-high'; 25,647 votes; published 95% interval 1192.9–1205.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-3",
    "bench": "lmarena-vision",
    "score": 1200.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-large-3'; 8,160 votes; published 95% interval 1192.4–1208.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "lmarena-vision",
    "score": 1200.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4.1-mini-2025-04-14'; 40,223 votes; published 95% interval 1193.3–1208.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o4-mini",
    "bench": "lmarena-vision",
    "score": 1200.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'o4-mini-2025-04-16'; 41,340 votes; published 95% interval 1193.7–1208.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "lmarena-vision",
    "score": 1199.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-medium-3.5'; 5,379 votes; published 95% interval 1189.9–1209.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-lite-preview-06-17",
    "bench": "lmarena-vision",
    "score": 1188.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-lite-preview-06-17-thinking'; 35,941 votes; published 95% interval 1180.9–1195.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-mini",
    "bench": "lmarena-vision",
    "score": 1183.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5-mini-high'; 28,965 votes; published 95% interval 1175.1–1191.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-3-27b",
    "bench": "lmarena-vision",
    "score": 1159.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemma-3-27b-it'; 16,989 votes; published 95% interval 1151.6–1167.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-nano",
    "bench": "lmarena-vision",
    "score": 1146.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5-nano-high'; 3,930 votes; published 95% interval 1134.9–1157.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-4-maverick",
    "bench": "lmarena-vision",
    "score": 1145.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llama-4-maverick-17b-128e-instruct'; 6,923 votes; published 95% interval 1136.1–1154.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "lmarena-vision",
    "score": 1139.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-small-2506'; 10,848 votes; published 95% interval 1130.7–1148.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-4-scout",
    "bench": "lmarena-vision",
    "score": 1127.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llama-4-scout-17b-16e-instruct'; 6,451 votes; published 95% interval 1117.9–1136.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5",
    "bench": "lmarena-document",
    "score": 1498.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-fable-5'; 8,497 votes; published 95% interval 1490.9–1506.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "lmarena-document",
    "score": 1495.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-opus-4-6-high'; 27,929 votes; published 95% interval 1488.8–1501.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "lmarena-document",
    "score": 1493,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-opus-4-7-high'; 21,957 votes; published 95% interval 1486.5–1499.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "lmarena-document",
    "score": 1492.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-fable-5.1-max'; 1,403 votes; published 95% interval 1477.5–1508.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-5",
    "bench": "lmarena-document",
    "score": 1489.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-opus-5-high'; 8,794 votes; published 95% interval 1482.6–1496.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "lmarena-document",
    "score": 1488.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.6-sol-xhigh'; 4,818 votes; published 95% interval 1479.8–1497.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "lmarena-document",
    "score": 1485.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-opus-4-8-high'; 11,551 votes; published 95% interval 1479.0–1492.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5",
    "bench": "lmarena-document",
    "score": 1480.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.5-high'; 20,944 votes; published 95% interval 1473.7–1486.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-1",
    "bench": "lmarena-document",
    "score": 1479.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'muse-spark-1.1'; 4,885 votes; published 95% interval 1470.5–1488.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "lmarena-document",
    "score": 1477.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-sonnet-4-6'; 57,813 votes; published 95% interval 1471.8–1483.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "lmarena-document",
    "score": 1476.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.6-terra-xhigh'; 5,787 votes; published 95% interval 1468.3–1485.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "lmarena-document",
    "score": 1475.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-sonnet-5-high'; 7,411 votes; published 95% interval 1468.0–1483.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-astra",
    "bench": "lmarena-document",
    "score": 1472.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-6-astra-max'; 1,621 votes; published 95% interval 1458.3–1486.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4",
    "bench": "lmarena-document",
    "score": 1470.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.4'; 33,331 votes; published 95% interval 1463.8–1476.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark-1-3",
    "bench": "lmarena-document",
    "score": 1467.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'muse-spark-1.3-max'; 1,006 votes; published 95% interval 1449.6–1486.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "lmarena-document",
    "score": 1464.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-opus-4-5-20251101'; 7,981 votes; published 95% interval 1454.2–1475.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-5",
    "bench": "lmarena-document",
    "score": 1464.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'grok-4.5'; 4,965 votes; published 95% interval 1455.5–1473.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "lmarena-document",
    "score": 1462,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.6-luna-xhigh'; 5,766 votes; published 95% interval 1453.6–1470.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-6",
    "bench": "lmarena-document",
    "score": 1461.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'grok-4.6-high'; 2,258 votes; published 95% interval 1449.0–1473.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "lmarena-document",
    "score": 1453.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-3.5-flash-high'; 4,061 votes; published 95% interval 1444.0–1462.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "lmarena-document",
    "score": 1458.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-3.1-pro-preview'; 49,457 votes; published 95% interval 1453.6–1464.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "lmarena-document",
    "score": 1453.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-3.6-flash-high'; 2,975 votes; published 95% interval 1442.5–1464.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-7-plus",
    "bench": "lmarena-document",
    "score": 1452.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'qwen3.7-plus'; 4,591 votes; published 95% interval 1443.5–1461.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-6",
    "bench": "lmarena-document",
    "score": 1449.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'kimi-k2.6'; 11,291 votes; published 95% interval 1442.1–1457.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "lmarena-document",
    "score": 1449.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-sonnet-4-5-20250929'; 31,455 votes; published 95% interval 1442.8–1455.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-4-31b",
    "bench": "lmarena-document",
    "score": 1444.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemma-4-31b'; 12,326 votes; published 95% interval 1436.7–1452.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20",
    "bench": "lmarena-document",
    "score": 1439.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'grok-4.20-beta-0309-reasoning'; 20,948 votes; published 95% interval 1432.3–1446.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-5",
    "bench": "lmarena-document",
    "score": 1436,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'kimi-k2.5-thinking'; 21,569 votes; published 95% interval 1429.1–1442.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-m3",
    "bench": "lmarena-document",
    "score": 1433,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'minimax-m3'; 6,314 votes; published 95% interval 1424.6–1441.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "lmarena-document",
    "score": 1430.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-2.5-pro'; 25,110 votes; published 95% interval 1424.2–1437.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2",
    "bench": "lmarena-document",
    "score": 1423.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.2-high'; 7,108 votes; published 95% interval 1413.7–1432.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "lmarena-document",
    "score": 1418.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'claude-haiku-4-5-20251001'; 34,677 votes; published 95% interval 1412.8–1424.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1",
    "bench": "lmarena-document",
    "score": 1415.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.1'; 8,244 votes; published 95% interval 1406.5–1425.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "lmarena-search",
    "score": 1255.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5.6-sol-xhigh'; 29,663 votes; published 95% interval 1248.1–1263.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-fable-5",
    "bench": "lmarena-search",
    "score": 1228.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-fable-5'; 41,795 votes; published 95% interval 1220.9–1236.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5",
    "bench": "lmarena-search",
    "score": 1223.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5.5-search'; 89,873 votes; published 95% interval 1218.1–1229.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "lmarena-search",
    "score": 1223.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-6-search'; 134,699 votes; published 95% interval 1218.2–1228.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "lmarena-search",
    "score": 1212,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-7'; 91,394 votes; published 95% interval 1206.5–1217.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "lmarena-search",
    "score": 1207.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gemini-3.1-pro-grounding'; 113,282 votes; published 95% interval 1202.4–1213.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-5",
    "bench": "lmarena-search",
    "score": 1202.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4.5'; 31,505 votes; published 95% interval 1195.4–1209.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "lmarena-search",
    "score": 1201,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-sonnet-4-6-search'; 134,905 votes; published 95% interval 1195.9–1206.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "lmarena-search",
    "score": 1196.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-sonnet-5-search'; 40,230 votes; published 95% interval 1189.1–1203.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "lmarena-search",
    "score": 1195,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-8'; 70,998 votes; published 95% interval 1188.6–1201.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-4",
    "bench": "lmarena-search",
    "score": 1194.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5.4-search'; 110,116 votes; published 95% interval 1188.9–1200.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-3",
    "bench": "lmarena-search",
    "score": 1193.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4.3'; 91,483 votes; published 95% interval 1187.9–1198.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o3",
    "bench": "lmarena-search",
    "score": 1191.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'o3-search'; 20,644 votes; published 95% interval 1185.1–1198.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "lmarena-search",
    "score": 1187.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-5-search'; 61,573 votes; published 95% interval 1181.5–1193.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1",
    "bench": "lmarena-search",
    "score": 1183.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5.1-search'; 59,909 votes; published 95% interval 1177.8–1188.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5",
    "bench": "lmarena-search",
    "score": 1182.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5-search'; 20,781 votes; published 95% interval 1175.4–1190.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "lmarena-search",
    "score": 1175.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-sonnet-4-5-search'; 127,378 votes; published 95% interval 1170.0–1180.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2",
    "bench": "lmarena-search",
    "score": 1161.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gpt-5.2-search'; 52,712 votes; published 95% interval 1155.6–1167.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "lmarena-search",
    "score": 1142.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gemini-2.5-pro-grounding'; 83,404 votes; published 95% interval 1137.4–1146.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "imagen-4",
    "bench": "lmarena-image",
    "score": 1129,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'imagen-4.0-generate-001'; 539,732 votes; published 95% interval 1126.3–1131.8; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1129.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-4-5",
    "bench": "lmarena-image",
    "score": 1148.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'seedream-4.5'; 300,107 votes; published 95% interval 1146.0–1151.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1147.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "lmarena-video",
    "score": 1516.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'gemini-omni-1.1-flash'; 1,784 votes; published 95% interval 1500.8–1531.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "lmarena-video-i2v",
    "score": 1488.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'gemini-omni-1.1-flash'; 3,734 votes; published 95% interval 1477.6–1499.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-3-video",
    "bench": "lmarena-video",
    "score": 1493.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'flux-3-video'; 1,302 votes; published 95% interval 1475.8–1510.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-5",
    "bench": "lmarena-video",
    "score": 1473.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'dreamina-seedance-2.5-720p'; 8,001 votes; published 95% interval 1465.4–1482.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-5",
    "bench": "lmarena-video-i2v",
    "score": 1477.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'dreamina-seedance-2.5-720p'; 15,729 votes; published 95% interval 1470.2–1484.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "p-video",
    "bench": "lmarena-video",
    "score": 1207.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'p-video'; 7,079 votes; published 95% interval 1191.6–1222.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "p-video",
    "bench": "lmarena-video-i2v",
    "score": 1243.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'p-video'; 23,364 votes; published 95% interval 1227.4–1259.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-6",
    "bench": "lmarena-video",
    "score": 1326.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'wan2.6-t2v'; 54,050 votes; published 95% interval 1319.3–1334.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-6",
    "bench": "lmarena-video-i2v",
    "score": 1309.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'wan2.6-i2v'; 121,082 votes; published 95% interval 1302.8–1316.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-5-preview",
    "bench": "lmarena-video",
    "score": 1245.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'wan2.5-t2v-preview'; 36,378 votes; published 95% interval 1236.7–1254.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-5-preview",
    "bench": "lmarena-video-i2v",
    "score": 1321.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'wan2.5-i2v-preview'; 16,907 votes; published 95% interval 1311.1–1331.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kandinsky-5-0-pro",
    "bench": "lmarena-video",
    "score": 1175.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kandinsky-5.0-t2v-pro'; 2,019 votes; published 95% interval 1154.4–1195.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "lmarena-video-i2v",
    "score": 1456,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'grok-imagine-video-1.5-720p'; 137,459 votes; published 95% interval 1451.8–1460.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "lmarena-text",
    "score": 1473.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.5-instant'; 27,098 votes; published 95% interval 1468.2–1478.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "lmarena-vision",
    "score": 1276,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.5-instant'; 7,605 votes; published 95% interval 1267.3–1284.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "lmarena-document",
    "score": 1460.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gpt-5.5-instant'; 8,497 votes; published 95% interval 1451.5–1469.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-1",
    "bench": "lmarena-text",
    "score": 1467.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ernie-5.1'; 39,396 votes; published 95% interval 1463.0–1472.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-1",
    "bench": "lmarena-search",
    "score": 1193.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'ernie-5.1'; 3,788 votes; published 95% interval 1183.7–1203.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5v-turbo",
    "bench": "lmarena-text",
    "score": 1433.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-5v-turbo'; 10,436 votes; published 95% interval 1427.0–1439.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5v-turbo",
    "bench": "lmarena-vision",
    "score": 1230.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'glm-5v-turbo'; 41,533 votes; published 95% interval 1224.6–1235.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5v-turbo",
    "bench": "lmarena-document",
    "score": 1400.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'glm-5v-turbo'; 6,995 votes; published 95% interval 1391.7–1408.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-5v-turbo",
    "bench": "lmarena-webdev",
    "score": 1399.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'glm-5v-turbo'; 2,416 votes; published 95% interval 1386.6–1413.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-omni",
    "bench": "lmarena-text",
    "score": 1431,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mimo-v2-omni'; 20,596 votes; published 95% interval 1425.5–1436.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-omni",
    "bench": "lmarena-vision",
    "score": 1216.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mimo-v2-omni'; 10,665 votes; published 95% interval 1208.7–1223.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark",
    "bench": "lmarena-text",
    "score": 1489.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'muse-spark'; 14,120 votes; published 95% interval 1483.6–1495.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark",
    "bench": "lmarena-vision",
    "score": 1293.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'muse-spark'; 5,868 votes; published 95% interval 1284.5–1302.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "muse-spark",
    "bench": "lmarena-document",
    "score": 1467.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'muse-spark'; 1,084 votes; published 95% interval 1448.9–1485.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "lmarena-webdev",
    "score": 1472.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'kimi-k2.7-code'; 4,993 votes; published 95% interval 1463.5–1482.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "laguna-m-1",
    "bench": "lmarena-webdev",
    "score": 1348.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'laguna-m.1'; 5,418 votes; published 95% interval 1338.8–1357.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "laguna-xs-2",
    "bench": "lmarena-webdev",
    "score": 1301.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'laguna-xs.2'; 4,256 votes; published 95% interval 1291.2–1312.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-1-8b",
    "bench": "lmarena-text",
    "score": 1304.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-4.1-8b'; 4,161 votes; published 95% interval 1294.0–1314.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-4-1-8b",
    "bench": "lmarena-webdev",
    "score": 1190.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'granite-4.1-8b'; 1,786 votes; published 95% interval 1171.9–1208.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "lmarena-webdev",
    "score": 1670,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.8-max-0902'; 9,485 votes; published 95% interval 1662.2–1677.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-2-5",
    "bench": "lmarena-image",
    "score": 1254.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'mai-image-2.5'; 61,265 votes; published 95% interval 1250.0–1258.3; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1254), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-2-0",
    "bench": "lmarena-image",
    "score": 1269.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'reve-2.0'; 15,773 votes; published 95% interval 1263.0–1275.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1268.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-2-1",
    "bench": "lmarena-image",
    "score": 1301.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'reve-2.1'; 8,249 votes; published 95% interval 1294.2–1309.2; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1302.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "krea-2-large",
    "bench": "lmarena-image",
    "score": 1111.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'krea-2-large'; 56,888 votes; published 95% interval 1107.4–1115.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1109.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "krea-2-medium",
    "bench": "lmarena-image",
    "score": 1124.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'krea-2-medium'; 58,098 votes; published 95% interval 1120.1–1128.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1123.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "krea-2-turbo",
    "bench": "lmarena-image",
    "score": 1113.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'krea-2-turbo'; 51,552 votes; published 95% interval 1108.7–1117.6; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1112.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-2-lite",
    "bench": "lmarena-image",
    "score": 1250.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-3.1-flash-lite-image (nano-banana-2-lite)'; 17,804 votes; published 95% interval 1244.3–1256.2; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1250), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-image-2-1",
    "bench": "lmarena-image",
    "score": 1222.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'qwen-image-2.1'; 11,441 votes; published 95% interval 1216.3–1229.6; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1227.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "muse-video",
    "bench": "lmarena-video",
    "score": 1455.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'muse-video'; 2,188 votes; published 95% interval 1441.2–1470.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hidream-o1-video-1-0",
    "bench": "lmarena-video-i2v",
    "score": 1456.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'hidream-o1-video-1.0'; 6,488 votes; published 95% interval 1448.5–1464.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "gpqa-diamond",
    "score": 0.5574494949494949,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-03-10",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Llama-70B'; run id f6pnAATjxmrFJS353EjQKi; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02973465219179067.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "otis-mock-aime",
    "score": 0.5138888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Llama-70B'; run id 89zcekqEYnE8zhwG4pR5Nu; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.060540263104731575.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "math-level-5",
    "score": 0.8989803625377644,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-10",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'DeepSeek-R1-Distill-Llama-70B'; run id oPDoTJFMhMNvU4sybYu2jQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006818015200174493.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "gpqa-diamond",
    "score": 0.8251262626262627,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-instant'; run id WPVhgi8NT9grAT7kfipT9G; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022532782662252126.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "otis-mock-aime",
    "score": 0.6805555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-instant'; run id hGzLQWbxPdgQiiMbaDqF8k; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.062127114361967634.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "frontiermath",
    "score": 0.2631578947368421,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-instant'; run id gGcHjE6ppy54ahN2pfxsGH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026129809832490088.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "epoch-chess",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-instant'; run id dodABAysqTvxvThpS84BpK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03265986323710904.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.024390243902439025,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-08-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-instant'; run id 2UmuBfhBvYbbnTg5BnQqov; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024390243902439022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-pro",
    "bench": "frontiermath",
    "score": 0.8771929824561403,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-pro_xhigh'; run id LSKDetgzyVojru45w7RJhf; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019476010341530077.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.7804878048780488,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.5-pro_xhigh'; run id SFKXCPUu73mwxMoUotiZM2; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06544589202438408.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "gpqa-diamond",
    "score": 0.8787878787878788,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id XneEig6pACkAWeekvgCwNk; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02325315795194205.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "otis-mock-aime",
    "score": 0.9555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id ENc5ZM5TpBYNKkTWN4xyQM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031067790907534736.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "frontiermath",
    "score": 0.5403508771929825,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id X3RxRkt2pBUXYJjAQHsUo6; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02957276813514758.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "simpleqa-verified",
    "score": 0.365,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id 5AijYw5bFJtSuahnbzNo6Z; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015231776226264846.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "epoch-chess",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id k7EnhoBbZyVFUVRBowA5Pm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-7-code",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.12195121951219512,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'kimi-k2.7-code'; run id UTVSgirxQpTgmTHbgE9EUk; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051739520574625435.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "gpqa-diamond",
    "score": 0.922979797979798,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id VzeQXKBH9maR2nDcZJ8b7S; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017249976996605708.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id 2f8Vn5QBdJnc9i5y9jzXS7; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "frontiermath",
    "score": 0.656140350877193,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id FpQf5xBwJPb4ZEo6SgdKWq; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02818576398907258.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "simpleqa-verified",
    "score": 0.473,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id nBcLJJomVoeg2zh5F8Mnsu; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01579621855130273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "epoch-chess",
    "score": 0.4,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id 8qGBfpdQXMmLcSqPwz7ofm; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0492365963917331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.34146341463414637,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id kZUdrqdGtCEJa3ip6L9YnH; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07497768853141171.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "epoch-furniture",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-09-25; exact model version 'qwen3.8-max-0902_xhigh'; run id kpiLkPjDgoKRYp2SiKm9Zo; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05207556439232955.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-flash",
    "bench": "gpqa-diamond",
    "score": 0.8232323232323232,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.7-flash'; run id i6i3gQf3ng2aAipvowPpws; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027178752639044908.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-flash",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.7-flash'; run id TjoS7sNuqVTc4LVyGRo4wa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-flash",
    "bench": "frontiermath",
    "score": 0.19298245614035087,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.7-flash'; run id 6pTYmQCw3EMbErvAqpivH5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023417537950655314.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-flash",
    "bench": "epoch-chess",
    "score": 0.23,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-flash'; run id 5Ax3kMh4Lmw8KsrJVWbXqn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04229525846816508.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-7-flash",
    "bench": "epoch-mystery-games",
    "score": 0.15,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.7-flash'; run id 5e4Jp9dxtNisWx2wbRRZxM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03588702812826367.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "gpqa-diamond",
    "score": 0.8333333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id H2FvRXagQRQmiNa7WrNMU2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026552207828215258.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "otis-mock-aime",
    "score": 0.8444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id 9zx2EXPTvPmxsL9qFCjP3x; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05463890236888291.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "frontiermath",
    "score": 0.17192982456140352,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id nDsNFM5ioGzyWcpeUKVotg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022389789548206734.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "simpleqa-verified",
    "score": 0.15915915915915915,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id FoMSS4FmH3VxpsvZiUJPCv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011579970321213651.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id g64kPFtWB7Adto8Y8Hi3XE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "epoch-mystery-games",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-flash'; run id 2y5jXmHGjyEyZvGjNDtRZe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03265986323710904.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark",
    "bench": "gpqa-diamond",
    "score": 0.898,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-04-08",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'muse-spark'; run id muse-spark-pre-release-gpqa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "muse-spark",
    "bench": "otis-mock-aime",
    "score": 0.889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-04-08",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'muse-spark'; run id muse-spark-pre-release-mock-aime; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.047.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "gpqa-diamond",
    "score": 0.8421717171717171,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-11-11",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'kimi-k2-thinking-turbo'; run id muGecFpAg9wHd6gexWJ99L; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020879845731985747. Epoch model_metadata.csv lists this version under model group 'Kimi K2 Thinking' (display name 'Kimi K2 Thinking Turbo', same release date and compute); it is the -turbo serving of K2 Thinking, matching the existing arena mapping for kimi-k2-thinking.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "otis-mock-aime",
    "score": 0.8305555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-11-13",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'kimi-k2-thinking-turbo'; run id 8BM7dCHaUZgcYJwZWiBhiQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048192643258048534. Epoch model_metadata.csv lists this version under model group 'Kimi K2 Thinking' (display name 'Kimi K2 Thinking Turbo', same release date and compute); it is the -turbo serving of K2 Thinking, matching the existing arena mapping for kimi-k2-thinking.",
    "source_id": "epoch-internal"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'kimi-k2-thinking-turbo'; run id 8JwFyEbMELziSdLLg8hatX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849. Epoch model_metadata.csv lists this version under model group 'Kimi K2 Thinking' (display name 'Kimi K2 Thinking Turbo', same release date and compute); it is the -turbo serving of K2 Thinking, matching the existing arena mapping for kimi-k2-thinking.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "gpqa-diamond",
    "score": 0.8342352092352093,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'deepseek-reasoner'; run id CDdjbqiYYPtEBGoZLMPM3n; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0201537212608744. Epoch model_metadata.csv lists this API alias as 'DeepSeek-V3.2 (Thinking)' (model group DeepSeek-V3.2, release 2025-12-01), and these runs started 2025-12-16, before DeepSeek V4. The non-thinking 'deepseek-chat' rows (run 2026-07-16) are not used.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "otis-mock-aime",
    "score": 0.878174603174603,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'deepseek-reasoner'; run id 8szQqvTsGG8gJ2TiFReRn9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.041306766974863386. Epoch model_metadata.csv lists this API alias as 'DeepSeek-V3.2 (Thinking)' (model group DeepSeek-V3.2, release 2025-12-01), and these runs started 2025-12-16, before DeepSeek V4. The non-thinking 'deepseek-chat' rows (run 2026-07-16) are not used.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "epoch-chess",
    "score": 0.14,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-reasoner'; run id CsCv6cEsjHtZnC6aoatVUC; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0348735088019777. Epoch model_metadata.csv lists this API alias as 'DeepSeek-V3.2 (Thinking)' (model group DeepSeek-V3.2, release 2025-12-01), and these runs started 2025-12-16, before DeepSeek V4. The non-thinking 'deepseek-chat' rows (run 2026-07-16) are not used.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-image-2-5-sunburst",
    "bench": "lmarena-image-edit",
    "score": 1523.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-2.5-sunburst'; 59,825 votes; published 95% interval 1518.2–1529.0; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1522.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-2-5-flare",
    "bench": "lmarena-image-edit",
    "score": 1481,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-2.5-flare'; 57,408 votes; published 95% interval 1475.7–1486.3; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1478.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-image-2",
    "bench": "lmarena-image-edit",
    "score": 1461.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-2 (medium)'; 305,825 votes; published 95% interval 1458.6–1464.9; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1461.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "grok-imagine-image-2",
    "bench": "lmarena-image-edit",
    "score": 1426.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'grok-imagine-image-2.0 (low)'; 36,022 votes; published 95% interval 1421.5–1432.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-2-6",
    "bench": "lmarena-image-edit",
    "score": 1428.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'mai-image-2.6'; 51,384 votes; published 95% interval 1423.3–1433.3; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1426.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "muse-image",
    "bench": "lmarena-image-edit",
    "score": 1402.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'muse-image'; 142,644 votes; published 95% interval 1398.4–1406.9; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1403.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-5-0-pro",
    "bench": "lmarena-image-edit",
    "score": 1393.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'seedream-5.0-pro'; 282,865 votes; published 95% interval 1390.5–1396.7; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1393.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-pro",
    "bench": "lmarena-image-edit",
    "score": 1386,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-3-pro-image-preview (nano-banana-pro)'; 540,775 votes; published 95% interval 1383.1–1388.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1385.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-2",
    "bench": "lmarena-image-edit",
    "score": 1387.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-3.1-flash-image (nano-banana-2) [web-search]'; 216,640 votes; published 95% interval 1384.3–1390.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-image-1-5",
    "bench": "lmarena-image-edit",
    "score": 1369.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-1.5-high-fidelity'; 654,249 votes; published 95% interval 1367.1–1372.3; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1369.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-4-5",
    "bench": "lmarena-image-edit",
    "score": 1301.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'seedream-4.5'; 1,294,217 votes; published 95% interval 1299.7–1304.0; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1301.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana",
    "bench": "lmarena-image-edit",
    "score": 1293.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-2.5-flash-image-preview (nano-banana)'; 11,383,435 votes; published 95% interval 1291.2–1295.2; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1293.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-4-0",
    "bench": "lmarena-image-edit",
    "score": 1270.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'seedream-4-2k'; 210,059 votes; published 95% interval 1263.5–1276.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1270.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-max",
    "bench": "lmarena-image-edit",
    "score": 1261.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-max'; 375,861 votes; published 95% interval 1258.5–1264.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1261.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-pro",
    "bench": "lmarena-image-edit",
    "score": 1245.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-pro'; 624,352 votes; published 95% interval 1242.8–1247.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-dev",
    "bench": "lmarena-image-edit",
    "score": 1224.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-dev'; 254,281 votes; published 95% interval 1220.7–1228.6; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1224.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "lmarena-image-edit",
    "score": 1224.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-klein-9b'; 540,833 votes; published 95% interval 1221.8–1227.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-klein-4b",
    "bench": "lmarena-image-edit",
    "score": 1187.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-klein-4b'; 541,422 votes; published 95% interval 1184.6–1190.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-image-1",
    "bench": "lmarena-image-edit",
    "score": 1138.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-1'; 2,870,020 votes; published 95% interval 1136.0–1141.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-image-1-mini",
    "bench": "lmarena-image-edit",
    "score": 1124.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'gpt-image-1-mini'; 682,950 votes; published 95% interval 1121.4–1127.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-flex",
    "bench": "lmarena-image-edit",
    "score": 1224.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-2-flex'; 443,953 votes; published 95% interval 1221.8–1228.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-kontext-max",
    "bench": "lmarena-image-edit",
    "score": 1180.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-1-kontext-max'; 390,877 votes; published 95% interval 1178.0–1183.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "lmarena-image-edit",
    "score": 1176.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'flux-1-kontext-pro'; 6,404,790 votes; published 95% interval 1173.8–1179.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "lmarena-image-edit",
    "score": 1149.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'flux-1-kontext-dev'; 3,639,808 votes; published 95% interval 1145.9–1152.5; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1149.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-image-edit",
    "bench": "lmarena-image-edit",
    "score": 1241.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'qwen-image-edit'; 2,022,077 votes; published 95% interval 1238.9–1244.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "lmarena-image-edit",
    "score": 1234,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'qwen-image-edit-2511'; 485,308 votes; published 95% interval 1231.3–1236.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "bagel",
    "bench": "lmarena-image-edit",
    "score": 1026.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'bagel'; 13,488 votes; published 95% interval 1021.0–1032.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seededit-3-0",
    "bench": "lmarena-image-edit",
    "score": 1138.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'seededit-3.0'; 4,918,196 votes; published 95% interval 1136.2–1141.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-image",
    "bench": "lmarena-image-edit",
    "score": 1080.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-2.0-flash-preview-image-generation'; 4,943,024 votes; published 95% interval 1078.5–1083.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuanimage-3-0-instruct",
    "bench": "lmarena-image-edit",
    "score": 1302.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'hunyuan-image-3.0-instruct'; 331,592 votes; published 95% interval 1299.4–1306.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-2-5",
    "bench": "lmarena-image-edit",
    "score": 1400.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'mai-image-2.5'; 190,081 votes; published 95% interval 1397.2–1404.6; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1400.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-2-1",
    "bench": "lmarena-image-edit",
    "score": 1374.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'reve-2.1'; 20,792 votes; published 95% interval 1368.2–1380.3; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1374.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-2-0",
    "bench": "lmarena-image-edit",
    "score": 1358.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'reve-2.0'; 19,265 votes; published 95% interval 1352.0–1365.0; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1358.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-v1-1",
    "bench": "lmarena-image-edit",
    "score": 1261.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'reve-v1.1'; 672,675 votes; published 95% interval 1259.1–1264.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reve-v1",
    "bench": "lmarena-image-edit",
    "score": 1234.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'reve-v1'; 395,314 votes; published 95% interval 1229.6–1239.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reve-v1-1-fast",
    "bench": "lmarena-image-edit",
    "score": 1207.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'reve-v1.1-fast'; 553,317 votes; published 95% interval 1204.2–1210.2; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1207.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "reve-edit-fast",
    "bench": "lmarena-image-edit",
    "score": 1198.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'reve-edit-fast'; 230,289 votes; published 95% interval 1194.0–1202.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "uni-1-1",
    "bench": "lmarena-image-edit",
    "score": 1315.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'uni-1.1'; 188,834 votes; published 95% interval 1311.9–1318.6; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1314.8), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "uni-1-1-max",
    "bench": "lmarena-image-edit",
    "score": 1334.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'uni-1.1-max'; 43,451 votes; published 95% interval 1329.3–1339.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nano-banana-2-lite",
    "bench": "lmarena-image-edit",
    "score": 1313.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-3.1-flash-lite-image (nano-banana-2-lite)'; 65,669 votes; published 95% interval 1308.8–1318.2; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1313.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-image-2-1",
    "bench": "lmarena-image-edit",
    "score": 1368.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'qwen-image-2.1'; 21,941 votes; published 95% interval 1362.8–1374.6; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1366.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "grok-imagine-image",
    "bench": "lmarena-image-edit",
    "score": 1329,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'grok-imagine-image'; 756,681 votes; published 95% interval 1326.5–1331.4; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1329.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "lmarena-image-edit",
    "score": 1294.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'seedream-5.0-lite'; 459,744 votes; published 95% interval 1291.5–1296.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1293.9), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "wan-2-7-image",
    "bench": "lmarena-image-edit",
    "score": 1301.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'wan2.7-image'; 44,814 votes; published 95% interval 1297.4–1305.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-7-image-pro",
    "bench": "lmarena-image-edit",
    "score": 1302.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'wan2.7-image-pro'; 43,992 votes; published 95% interval 1298.6–1306.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-6-image",
    "bench": "lmarena-image-edit",
    "score": 1230.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'wan2.6-image'; 664,189 votes; published 95% interval 1228.0–1233.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-5-i2i-preview",
    "bench": "lmarena-image-edit",
    "score": 1181.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'wan2.5-i2i-preview'; 723,697 votes; published 95% interval 1178.5–1183.7; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1180.6), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "kling-image-o1",
    "bench": "lmarena-image-edit",
    "score": 1251.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'kling-image-o1'; 143,692 votes; published 95% interval 1247.5–1254.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-2-flex",
    "bench": "lmarena-image",
    "score": 1156.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-2-flex'; 156,923 votes; published 95% interval 1153.2–1159.9; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1156.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-1-kontext-max",
    "bench": "lmarena-image",
    "score": 1074.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-1-kontext-max'; 65,517 votes; published 95% interval 1071.3–1077.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "lmarena-image",
    "score": 1058.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-1-kontext-pro'; 332,376 votes; published 95% interval 1055.3–1062.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "lmarena-image",
    "score": 941.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-1-kontext-dev'; 216,238 votes; published 95% interval 937.3–944.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "bagel",
    "bench": "lmarena-image",
    "score": 897.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'bagel'; 12,363 votes; published 95% interval 892.2–903.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-image",
    "bench": "lmarena-image",
    "score": 975.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-2.0-flash-preview-image-generation'; 257,241 votes; published 95% interval 971.9–978.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-2",
    "bench": "lmarena-image",
    "score": 1183.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'mai-image-2'; 51,477 votes; published 95% interval 1179.3–1188.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mai-image-1",
    "bench": "lmarena-image",
    "score": 1092.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'mai-image-1'; 98,414 votes; published 95% interval 1089.1–1096.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reve-v1-5",
    "bench": "lmarena-image",
    "score": 1155.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'reve-v1.5'; 39,755 votes; published 95% interval 1151.1–1159.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "uni-1-1",
    "bench": "lmarena-image",
    "score": 1184.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'uni-1.1'; 46,699 votes; published 95% interval 1179.9–1188.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1183.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "uni-1-1-max",
    "bench": "lmarena-image",
    "score": 1189.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'uni-1.1-max'; 14,072 votes; published 95% interval 1183.1–1195.5; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1189.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "grok-imagine-image",
    "bench": "lmarena-image",
    "score": 1169.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'grok-imagine-image'; 272,085 votes; published 95% interval 1167.0–1172.3; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1170.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "grok-imagine-image-pro",
    "bench": "lmarena-image",
    "score": 1162.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'grok-imagine-image-pro'; 99,455 votes; published 95% interval 1158.6–1166.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1162.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "lmarena-image",
    "score": 1137.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'seedream-5.0-lite'; 129,950 votes; published 95% interval 1134.7–1141.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1138.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "wan-2-7-image",
    "bench": "lmarena-image",
    "score": 1101.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'wan2.7-image'; 30,630 votes; published 95% interval 1096.1–1106.1; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1101.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "wan-2-7-image-pro",
    "bench": "lmarena-image",
    "score": 1103.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'wan2.7-image-pro'; 30,229 votes; published 95% interval 1098.5–1108.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-2-5-t2i-preview",
    "bench": "lmarena-image",
    "score": 1115.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'wan2.5-t2i-preview'; 279,229 votes; published 95% interval 1112.9–1118.4; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1116.2), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "recraft-v4-1-pro",
    "bench": "lmarena-image",
    "score": 1133,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'recraft-v4.1-pro'; 2,795 votes; published 95% interval 1122.6–1143.5; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1132.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "recraft-v4-1-utility-pro",
    "bench": "lmarena-image",
    "score": 1169.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'recraft-v4.1-utility-pro'; 2,618 votes; published 95% interval 1158.3–1180.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "recraft-v4-1-flash",
    "bench": "lmarena-image",
    "score": 1099.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'recraft-v4.1-flash'; 3,516 votes; published 95% interval 1090.0–1109.0; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1100.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "recraft-v3",
    "bench": "lmarena-image",
    "score": 1020.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'recraft-v3'; 195,627 votes; published 95% interval 1017.0–1024.3; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1020.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "seedream-3-0",
    "bench": "lmarena-image",
    "score": 1081.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'seedream-3'; 36,885 votes; published 95% interval 1076.6–1086.6; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1081.5), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "imagen-3",
    "bench": "lmarena-image",
    "score": 1058,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'imagen-3.0-generate-002'; 359,786 votes; published 95% interval 1054.8–1061.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "luma-photon",
    "bench": "lmarena-image",
    "score": 1035.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'photon'; 130,158 votes; published 95% interval 1032.0–1039.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "runway-gen-4-image",
    "bench": "lmarena-image",
    "score": 1024.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'runway-gen4'; 54,361 votes; published 95% interval 1019.8–1028.6; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1024.3), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "flux-1-1-pro",
    "bench": "lmarena-image",
    "score": 1015.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'flux-1.1-pro'; 70,460 votes; published 95% interval 1012.1–1019.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ideogram-2",
    "bench": "lmarena-image",
    "score": 1013.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'ideogram-v2'; 72,090 votes; published 95% interval 1009.8–1016.9; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1013.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "lucid-origin",
    "bench": "lmarena-image",
    "score": 1013,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'lucid-origin'; 287,345 votes; published 95% interval 1009.7–1016.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "dall-e-3",
    "bench": "lmarena-image",
    "score": 967.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_image/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-24",
    "note": "text_to_image, category overall, split latest; exact entry 'dall-e-3'; 238,749 votes; published 95% interval 964.1–971.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-fast",
    "bench": "lmarena-video",
    "score": 1347.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'veo-3-fast-audio'; 25,258 votes; published 95% interval 1336.6–1359.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-o1-pro",
    "bench": "lmarena-video",
    "score": 1204.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kling-o1-pro'; 1,194 votes; published 95% interval 1177.9–1231.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-0-pro",
    "bench": "lmarena-video",
    "score": 1190.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'seedance-v1-pro'; 12,164 votes; published 95% interval 1179.4–1202.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-0-lite",
    "bench": "lmarena-video",
    "score": 1112.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'seedance-v1-lite'; 16,253 votes; published 95% interval 1103.0–1122.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kandinsky-5-0-t2v-lite",
    "bench": "lmarena-video",
    "score": 1112.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kandinsky-5.0-t2v-lite'; 1,474 votes; published 95% interval 1094.7–1130.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-2",
    "bench": "lmarena-video",
    "score": 1164.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'veo-2'; 6,531 votes; published 95% interval 1148.1–1180.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-2-1-master",
    "bench": "lmarena-video",
    "score": 1162.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kling-v2.1-master'; 14,104 votes; published 95% interval 1153.0–1172.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sora",
    "bench": "lmarena-video",
    "score": 1068.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'sora'; 4,082 votes; published 95% interval 1052.6–1085.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "luma-ray-2",
    "bench": "lmarena-video",
    "score": 1065.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'ray2'; 5,250 votes; published 95% interval 1048.1–1082.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-3-fast",
    "bench": "lmarena-video-i2v",
    "score": 1326,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'veo-3-fast-audio'; 41,194 votes; published 95% interval 1316.5–1335.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-0-pro",
    "bench": "lmarena-video-i2v",
    "score": 1273,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'seedance-v1-pro'; 34,019 votes; published 95% interval 1265.2–1280.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-1-0-lite",
    "bench": "lmarena-video-i2v",
    "score": 1185.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'seedance-v1-lite'; 33,734 votes; published 95% interval 1177.5–1193.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "veo-2",
    "bench": "lmarena-video-i2v",
    "score": 1165.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'veo-2'; 10,303 votes; published 95% interval 1149.6–1182.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-2-1-master",
    "bench": "lmarena-video-i2v",
    "score": 1235,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'kling-v2.1-master'; 29,824 votes; published 95% interval 1227.1–1242.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-2-1-standard",
    "bench": "lmarena-video-i2v",
    "score": 1228.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'kling-v2.1-standard'; 29,957 votes; published 95% interval 1220.3–1237.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "luma-ray-2",
    "bench": "lmarena-video-i2v",
    "score": 1108.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'ray2'; 9,521 votes; published 95% interval 1092.2–1124.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "vidu-q2-turbo",
    "bench": "lmarena-video-i2v",
    "score": 1243.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'vidu-q2-turbo'; 2,509 votes; published 95% interval 1226.8–1260.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "vidu-q2-pro",
    "bench": "lmarena-video-i2v",
    "score": 1223.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'vidu-q2-pro'; 2,614 votes; published 95% interval 1206.6–1240.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "runway-gen-4-turbo",
    "bench": "lmarena-video-i2v",
    "score": 1052.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'runway-gen4-turbo'; 6,801 votes; published 95% interval 1039.0–1065.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wan-3-0",
    "bench": "lmarena-video-edit",
    "score": 1414.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'wan3.0'; 463 votes; published 95% interval 1388.4–1439.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-5",
    "bench": "lmarena-video-edit",
    "score": 1409.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'dreamina-seedance-2.5-720p'; 429 votes; published 95% interval 1383.3–1436.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minimax-h3",
    "bench": "lmarena-video-edit",
    "score": 1392.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'minimax-h3'; 962 votes; published 95% interval 1373.6–1410.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-omni-flash",
    "bench": "lmarena-video-edit",
    "score": 1366.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'gemini-omni-flash'; 2,109 votes; published 95% interval 1352.0–1381.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seedance-2-0",
    "bench": "lmarena-video-edit",
    "score": 1364.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'dreamina-seedance-2.0-720p'; 3,852 votes; published 95% interval 1350.3–1378.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "happyhorse-1-0",
    "bench": "lmarena-video-edit",
    "score": 1306.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'happyhorse-1.0'; 3,116 votes; published 95% interval 1293.1–1320.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-imagine-video",
    "bench": "lmarena-video-edit",
    "score": 1257.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'grok-imagine-video'; 13,724 votes; published 95% interval 1247.5–1268.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-o3-pro",
    "bench": "lmarena-video-edit",
    "score": 1255.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'kling-o3-pro'; 7,540 votes; published 95% interval 1243.9–1266.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kling-o1-pro",
    "bench": "lmarena-video-edit",
    "score": 1197.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'kling-o1-pro'; 10,542 votes; published 95% interval 1186.9–1207.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "runway-aleph",
    "bench": "lmarena-video-edit",
    "score": 1182.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/video_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-27",
    "note": "video_edit, category overall, split latest; exact entry 'runway-gen4-aleph'; 9,818 votes; published 95% interval 1173.5–1190.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nano-banana-pro",
    "bench": "genexam-relaxed",
    "score": 0.93,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Nano Banana Pro; S3.T2; relaxed Overall 93.0/100 (strict Overall 70.2, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run. Ref: Google 'Introducing Nano Banana Pro'.",
    "evidence": {
      "source_model_label": "Nano Banana Pro",
      "table": "S3.T2",
      "reported_value": "93.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Nano Banana Pro",
        "70.2",
        "93.0"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "genexam-relaxed",
    "score": 0.815,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "GPT-Image-1.5; S3.T2; relaxed Overall 81.5/100 (strict Overall 42.5, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "GPT-Image-1.5",
      "table": "S3.T2",
      "reported_value": "81.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "GPT-Image-1.5",
        "42.5",
        "81.5"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "gpt-image-1",
    "bench": "genexam-relaxed",
    "score": 0.622,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "GPT-Image-1; S3.T2; relaxed Overall 62.2/100 (strict Overall 13.1, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run. Ref: OpenAI image generation API (gpt-image-1).",
    "evidence": {
      "source_model_label": "GPT-Image-1",
      "table": "S3.T2",
      "reported_value": "62.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "GPT-Image-1",
        "13.1",
        "62.2"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "seedream-4-5",
    "bench": "genexam-relaxed",
    "score": 0.595,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Seedream 4.5; S3.T2; relaxed Overall 59.5/100 (strict Overall 12.3, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Seedream 4.5",
      "table": "S3.T2",
      "reported_value": "59.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Seedream 4.5",
        "12.3",
        "59.5"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "flux-2-max",
    "bench": "genexam-relaxed",
    "score": 0.616,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "FLUX.2 max; S3.T2; relaxed Overall 61.6/100 (strict Overall 8.6, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "FLUX.2 max",
      "table": "S3.T2",
      "reported_value": "61.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "FLUX.2 max",
        "8.6",
        "61.6"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "imagen-4-ultra",
    "bench": "genexam-relaxed",
    "score": 0.53,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Imagen-4-Ultra; S3.T2; relaxed Overall 53.0/100 (strict Overall 7.8, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Imagen-4-Ultra",
      "table": "S3.T2",
      "reported_value": "53.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Imagen-4-Ultra",
        "7.8",
        "53.0"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "flux-2-dev",
    "bench": "genexam-relaxed",
    "score": 0.423,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "FLUX.2 dev; S3.T2; relaxed Overall 42.3/100 (strict Overall 2.4, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "FLUX.2 dev",
      "table": "S3.T2",
      "reported_value": "42.3",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "FLUX.2 dev",
        "2.4",
        "42.3"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "qwen-image-2512",
    "bench": "genexam-relaxed",
    "score": 0.353,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Qwen-Image-2512; S3.T2; relaxed Overall 35.3/100 (strict Overall 1.5, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Qwen-Image-2512",
      "table": "S3.T2",
      "reported_value": "35.3",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Qwen-Image-2512",
        "1.5",
        "35.3"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "hidream-i1-full",
    "bench": "genexam-relaxed",
    "score": 0.212,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "HiDream-I1-Full; S3.T2; relaxed Overall 21.2/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "HiDream-I1-Full",
      "table": "S3.T2",
      "reported_value": "21.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "HiDream-I1-Full",
        "0.0",
        "21.2"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "sd-3-5-large",
    "bench": "genexam-relaxed",
    "score": 0.159,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Stable Diffusion 3.5 Large; S3.T2; relaxed Overall 15.9/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Stable Diffusion 3.5 Large",
      "table": "S3.T2",
      "reported_value": "15.9",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Stable Diffusion 3.5 Large",
        "0.0",
        "15.9"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "bagel",
    "bench": "genexam-relaxed",
    "score": 0.129,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "BAGEL (thinking); S3.T2; relaxed Overall 12.9/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run. Setting: thinking mode (non-thinking row 11.4 not recorded).",
    "evidence": {
      "source_model_label": "BAGEL (thinking)",
      "table": "S3.T2",
      "reported_value": "12.9",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "BAGEL (thinking)",
        "0.0",
        "12.9"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "show-o2-7b",
    "bench": "genexam-relaxed",
    "score": 0.112,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Show-o2-7B; S3.T2; relaxed Overall 11.2/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Show-o2-7B",
      "table": "S3.T2",
      "reported_value": "11.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Show-o2-7B",
        "0.0",
        "11.2"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "show-o2-1-5b-hq",
    "bench": "genexam-relaxed",
    "score": 0.101,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Show-o2-1.5B-HQ; S3.T2; relaxed Overall 10.1/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Show-o2-1.5B-HQ",
      "table": "S3.T2",
      "reported_value": "10.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Show-o2-1.5B-HQ",
        "0.0",
        "10.1"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "blip3o-next-grpo-text-3b",
    "bench": "genexam-relaxed",
    "score": 0.126,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "BLIP3o-NEXT-GRPO-Text-3B; S3.T2; relaxed Overall 12.6/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "BLIP3o-NEXT-GRPO-Text-3B",
      "table": "S3.T2",
      "reported_value": "12.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "BLIP3o-NEXT-GRPO-Text-3B",
        "0.0",
        "12.6"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "blip3o-8b",
    "bench": "genexam-relaxed",
    "score": 0.067,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "BLIP3o-8B; S3.T2; relaxed Overall 6.7/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "BLIP3o-8B",
      "table": "S3.T2",
      "reported_value": "6.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "BLIP3o-8B",
        "0.0",
        "6.7"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "emu3",
    "bench": "genexam-relaxed",
    "score": 0.088,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Emu3; S3.T2; relaxed Overall 8.8/100 (strict Overall 0.0, not recorded separately). GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
    "evidence": {
      "source_model_label": "Emu3",
      "table": "S3.T2",
      "reported_value": "8.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenExam v5 Table 2 relaxed Overall; default T2I configuration per model; GPT-5 judge (low reasoning effort); benchmark authors' run.",
      "source_row": [
        "Emu3",
        "0.0",
        "8.8"
      ],
      "source_row_columns": "Overall Str, Overall Rel (last two cells of the printed row)",
      "date_basis": "arXiv v5 submission date (14 May 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited release page; API snapshot, resolution and quality tier not disclosed ('default T2I configuration').",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We select 17 representative models for comparison, including closed-source proprietary models, open-source T2I models and unified MLLMs. We use the default T2I configuration for each of the models for inference.\"",
      "evaluator_relationship": "benchmark authors (Shanghai AI Laboratory / SJTU et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory et al.)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "sd-3-medium",
    "bench": "t2i-reasonbench-overall",
    "score": 0.521,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "SD-3-Medium; Table 2 (PDF p.6, proceedings p.8924); Overall 52.1/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "SD-3-Medium",
      "table": "Table 2",
      "reported_value": "52.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "SD-3-Medium",
        "41.7",
        "66.9",
        "43.1",
        "56.6",
        "52.1"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "sd-3-5-medium",
    "bench": "t2i-reasonbench-overall",
    "score": 0.522,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "SD-3.5-Medium; Table 2 (PDF p.6, proceedings p.8924); Overall 52.2/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "SD-3.5-Medium",
      "table": "Table 2",
      "reported_value": "52.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "SD-3.5-Medium",
        "39.6",
        "66.0",
        "47.6",
        "55.7",
        "52.2"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "sd-3-5-large",
    "bench": "t2i-reasonbench-overall",
    "score": 0.552,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "SD-3.5-Large; Table 2 (PDF p.6, proceedings p.8924); Overall 55.2/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "SD-3.5-Large",
      "table": "Table 2",
      "reported_value": "55.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "SD-3.5-Large",
        "44.2",
        "66.5",
        "50.9",
        "59.3",
        "55.2"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "flux-1-dev",
    "bench": "t2i-reasonbench-overall",
    "score": 0.549,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "FLUX.1-dev; Table 2 (PDF p.6, proceedings p.8924); Overall 54.9/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "FLUX.1-dev",
      "table": "Table 2",
      "reported_value": "54.9",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "FLUX.1-dev",
        "49.3",
        "67.9",
        "50.5",
        "51.8",
        "54.9"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "flux-1-schnell",
    "bench": "t2i-reasonbench-overall",
    "score": 0.576,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "FLUX.1-schnell; Table 2 (PDF p.6, proceedings p.8924); Overall 57.6/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "FLUX.1-schnell",
      "table": "Table 2",
      "reported_value": "57.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "FLUX.1-schnell",
        "47.4",
        "74.2",
        "50.1",
        "58.7",
        "57.6"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "playground-v2-5",
    "bench": "t2i-reasonbench-overall",
    "score": 0.505,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Playground-v2.5; Table 2 (PDF p.6, proceedings p.8924); Overall 50.5/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "Playground-v2.5",
      "table": "Table 2",
      "reported_value": "50.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Playground-v2.5",
        "50.8",
        "45.5",
        "51.1",
        "54.4",
        "50.5"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "hidream-i1-full",
    "bench": "t2i-reasonbench-overall",
    "score": 0.642,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "HiDream-I1-full; Table 2 (PDF p.6, proceedings p.8924); Overall 64.2/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "HiDream-I1-full",
      "table": "Table 2",
      "reported_value": "64.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "HiDream-I1-full",
        "59.1",
        "80.3",
        "58.0",
        "59.4",
        "64.2"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "qwen-image",
    "bench": "t2i-reasonbench-overall",
    "score": 0.675,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Qwen-Image; Table 2 (PDF p.6, proceedings p.8924); Overall 67.5/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "Qwen-Image",
      "table": "Table 2",
      "reported_value": "67.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Qwen-Image",
        "62.7",
        "77.1",
        "61.8",
        "68.2",
        "67.5"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "emu3",
    "bench": "t2i-reasonbench-overall",
    "score": 0.417,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Emu3; Table 2 (PDF p.6, proceedings p.8924); Overall 41.7/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "Emu3",
      "table": "Table 2",
      "reported_value": "41.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Emu3",
        "39.2",
        "40.2",
        "42.7",
        "44.6",
        "41.7"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "show-o-demo-512",
    "bench": "t2i-reasonbench-overall",
    "score": 0.428,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "show-o-demo-512; Table 2 (PDF p.6, proceedings p.8924); Overall 42.8/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "show-o-demo-512",
      "table": "Table 2",
      "reported_value": "42.8",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "show-o-demo-512",
        "38.9",
        "43.2",
        "41.0",
        "48.3",
        "42.8"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "janus-pro-7b",
    "bench": "t2i-reasonbench-overall",
    "score": 0.423,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Janus-Pro-7B; Table 2 (PDF p.6, proceedings p.8924); Overall 42.3/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
    "evidence": {
      "source_model_label": "Janus-Pro-7B",
      "table": "Table 2",
      "reported_value": "42.3",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Janus-Pro-7B",
        "32.7",
        "42.9",
        "41.3",
        "52.4",
        "42.3"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "bagel",
    "bench": "t2i-reasonbench-overall",
    "score": 0.556,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Bagel w/ Thinking; Table 2 (PDF p.6, proceedings p.8924); Overall 55.6/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column. Setting: thinking mode enabled (the paper's only Table 2 BAGEL configuration).",
    "evidence": {
      "source_model_label": "Bagel w/ Thinking",
      "table": "Table 2",
      "reported_value": "55.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Bagel w/ Thinking",
        "50.6",
        "50.5",
        "56.5",
        "64.9",
        "55.6"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "gpt-image-1",
    "bench": "t2i-reasonbench-overall",
    "score": 0.863,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "GPT-Image-1; Table 2 (PDF p.6, proceedings p.8924); Overall 86.3/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column. Reference: OpenAI gpt-image-1 image generation API.",
    "evidence": {
      "source_model_label": "GPT-Image-1",
      "table": "Table 2",
      "reported_value": "86.3",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "GPT-Image-1",
        "84.1",
        "92.4",
        "85.7",
        "82.9",
        "86.3"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "nano-banana",
    "bench": "t2i-reasonbench-overall",
    "score": 0.895,
    "source": "https://aclanthology.org/2026.findings-acl.433.pdf#page=6",
    "source_id": "visual-t2i-reasonbench",
    "by": "independent",
    "date": "2026-07-01",
    "note": "Nano-Banana; Table 2 (PDF p.6, proceedings p.8924); Overall 89.5/100. T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column. Reference: Google 2025 'Introducing Gemini 2.5 Flash Image'.",
    "evidence": {
      "source_model_label": "Nano-Banana",
      "table": "Table 2",
      "reported_value": "89.5",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "T2I-ReasonBench Table 2; original prompts; authors' default generation settings for all models; GPT-5.1/Gemini-2.5-pro question generation and Gemini-2.5-pro judge; Overall column.",
      "source_row": [
        "Nano-Banana",
        "89.8",
        "94.8",
        "87.3",
        "86.0",
        "89.5"
      ],
      "date_basis": "Findings of ACL 2026 proceedings (bib month=jul, year=2026); day not stated, first of month used. Not a generation date.",
      "model_identity_scope": "Exact source-reported release/checkpoint label; no API snapshot, resolution or seed claimed beyond 'default settings'.",
      "original_measurement_verified": true,
      "provenance_evidence": "Section 5: \"We evaluate 16 state-of-the-art T2I models, including 8 diffusion models, 5 unified multimodal models, and 3 proprietary models ... Implementation Details. We use the default settings for all T2I models in image generation.\"",
      "evaluator_relationship": "benchmark authors (HKU/CUHK/Huawei Noah's Ark Lab) evaluating other providers' models"
    },
    "evaluator": "T2I-ReasonBench authors (HKU / CUHK / Huawei Noah's Ark Lab)",
    "source_group": "visual-t2i-reasonbench:t2i-reasonbench-overall"
  },
  {
    "model": "qwen-image",
    "bench": "tiif-bench-v3-short",
    "score": 0.87,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Qwen-Image; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 87.5 (same run, not recorded separately). Label 'Qwen-Image' citing the Qwen-Image technical report; mapped to the original Qwen/Qwen-Image release, not Qwen-Image-2512.",
    "evidence": {
      "source_model_label": "Qwen-Image",
      "table": "S4.T2",
      "reported_value": "87.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "Qwen-Image",
        "87.0",
        "87.5"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "flux-2-dev",
    "bench": "tiif-bench-v3-short",
    "score": 0.872,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "FLUX.2 dev; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 87.3 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "FLUX.2 dev",
      "table": "S4.T2",
      "reported_value": "87.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "FLUX.2 dev",
        "87.2",
        "87.3"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "flux-1-dev",
    "bench": "tiif-bench-v3-short",
    "score": 0.726,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "FLUX.1 dev; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 71.0 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "FLUX.1 dev",
      "table": "S4.T2",
      "reported_value": "72.6",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "FLUX.1 dev",
        "72.6",
        "71.0"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "lumina-image-2",
    "bench": "tiif-bench-v3-short",
    "score": 0.712,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "LuminaImage2; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 70.7 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "LuminaImage2",
      "table": "S4.T2",
      "reported_value": "71.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "LuminaImage2",
        "71.2",
        "70.7"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "bagel",
    "bench": "tiif-bench-v3-short",
    "score": 0.71,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Bagel; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 71.3 (same run, not recorded separately). Thinking/CoT setting not stated.",
    "evidence": {
      "source_model_label": "Bagel",
      "table": "S4.T2",
      "reported_value": "71.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "Bagel",
        "71.0",
        "71.3"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "emu3-5",
    "bench": "tiif-bench-v3-short",
    "score": 0.859,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Emu3.5; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 84.8 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "Emu3.5",
      "table": "S4.T2",
      "reported_value": "85.9",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "Emu3.5",
        "85.9",
        "84.8"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "emu3-5-image",
    "bench": "tiif-bench-v3-short",
    "score": 0.884,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Emu3.5-Image; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 87.1 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "Emu3.5-Image",
      "table": "S4.T2",
      "reported_value": "88.4",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "Emu3.5-Image",
        "88.4",
        "87.1"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "dall-e-3",
    "bench": "tiif-bench-v3-short",
    "score": 0.722,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "DALL-E 3; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 70.2 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "DALL-E 3",
      "table": "S4.T2",
      "reported_value": "72.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "DALL-E 3",
        "72.2",
        "70.2"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "midjourney-v7",
    "bench": "tiif-bench-v3-short",
    "score": 0.677,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "MidJourney v7; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 62.7 (same run, not recorded separately). Appendix A4: MidJourney v7 used in the main benchmark tables.",
    "evidence": {
      "source_model_label": "MidJourney v7",
      "table": "S4.T2",
      "reported_value": "67.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "MidJourney v7",
        "67.7",
        "62.7"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "gpt-image-1",
    "bench": "tiif-bench-v3-short",
    "score": 0.861,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "GPT-Image-1; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 87.6 (same run, not recorded separately).",
    "evidence": {
      "source_model_label": "GPT-Image-1",
      "table": "S4.T2",
      "reported_value": "86.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "GPT-Image-1",
        "86.1",
        "87.6"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "nano-banana",
    "bench": "tiif-bench-v3-short",
    "score": 0.892,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Nano-Banana; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 88.5 (same run, not recorded separately). Ref [19]: Gemini 2.5 Flash Image.",
    "evidence": {
      "source_model_label": "Nano-Banana",
      "table": "S4.T2",
      "reported_value": "89.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "Nano-Banana",
        "89.2",
        "88.5"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "nano-banana-pro",
    "bench": "tiif-bench-v3-short",
    "score": 0.931,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T2",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "NB-Pro; S4.T2; TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated. Long-prompt Overall 93.3 (same run, not recorded separately). Ref [20]: nano-banana-pro (Gemini 3 Pro Image).",
    "evidence": {
      "source_model_label": "NB-Pro",
      "table": "S4.T2",
      "reported_value": "93.1",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TIIF-Bench v3 Table 2 short-prompt Overall; TIIF-Evaluator; benchmark authors' run; latest available model versions at evaluation time (Appendix A4); split not stated.",
      "source_row": [
        "NB-Pro",
        "93.1",
        "93.3"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:tiif-bench-v3-short"
  },
  {
    "model": "dall-e-3",
    "bench": "genai-bench-vqascore",
    "score": 0.81,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T3",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "DALLE-3[5]; S4.T3; GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
    "evidence": {
      "source_model_label": "DALLE-3[5]",
      "table": "S4.T3",
      "reported_value": "0.81",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
      "source_row": [
        "DALLE-3[5]",
        "0.85",
        "0.76",
        "0.81"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:genai-bench-vqascore"
  },
  {
    "model": "midjourney-v7",
    "bench": "genai-bench-vqascore",
    "score": 0.75,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T3",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "MidJourney v7[54]; S4.T3; GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
    "evidence": {
      "source_model_label": "MidJourney v7[54]",
      "table": "S4.T3",
      "reported_value": "0.75",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
      "source_row": [
        "MidJourney v7[54]",
        "0.85",
        "0.68",
        "0.75"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:genai-bench-vqascore"
  },
  {
    "model": "nano-banana",
    "bench": "genai-bench-vqascore",
    "score": 0.79,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T3",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "Nano-Banana[19]; S4.T3; GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
    "evidence": {
      "source_model_label": "Nano-Banana[19]",
      "table": "S4.T3",
      "reported_value": "0.79",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
      "source_row": [
        "Nano-Banana[19]",
        "0.84",
        "0.74",
        "0.79"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:genai-bench-vqascore"
  },
  {
    "model": "gpt-image-1",
    "bench": "genai-bench-vqascore",
    "score": 0.85,
    "source": "https://arxiv.org/html/2506.02161v3#S4.T3",
    "source_id": "visual-tiif-bench",
    "by": "independent",
    "date": "2026-07-12",
    "note": "GPT-Image-1[27]; S4.T3; GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
    "evidence": {
      "source_model_label": "GPT-Image-1[27]",
      "table": "S4.T3",
      "reported_value": "0.85",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GenAI-Bench Overall VQAScore, complete prompt set, official protocol, run by TIIF-Bench authors (v3 Table 3).",
      "source_row": [
        "GPT-Image-1[27]",
        "0.86",
        "0.83",
        "0.85"
      ],
      "date_basis": "arXiv v3 submission date (12 Jul 2026), not a generation date.",
      "model_identity_scope": "Exact source-reported model label with its cited reference; API snapshot, resolution and seeds not disclosed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We benchmark TIIF-Bench on a diverse suite of widely used closed-source and leading open-source text-to-image models. The results are summarized in Tab. 2\"; Sec. 4.2: \"For every benchmark, all models are evaluated on the complete prompt set using the benchmark's official evaluation protocol.\"",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "TIIF-Bench authors",
    "source_group": "visual-tiif-bench:genai-bench-vqascore"
  },
  {
    "model": "flux-1-dev",
    "bench": "wise-wiscore-overall",
    "score": 0.5,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "FLUX.1-dev; Table 1 (S4.T1); Overall 0.50. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "FLUX.1-dev",
      "table": "S4.T1",
      "reported_value": "0.50",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "FLUX.1-dev",
        "0.48",
        "0.58",
        "0.62",
        "0.42",
        "0.51",
        "0.35",
        "0.50"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "flux-1-schnell",
    "bench": "wise-wiscore-overall",
    "score": 0.4,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "FLUX.1-schnell; Table 1 (S4.T1); Overall 0.40. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "FLUX.1-schnell",
      "table": "S4.T1",
      "reported_value": "0.40",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "FLUX.1-schnell",
        "0.39",
        "0.44",
        "0.50",
        "0.31",
        "0.44",
        "0.26",
        "0.40"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "pixart-alpha",
    "bench": "wise-wiscore-overall",
    "score": 0.47,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "PixArt-Alpha; Table 1 (S4.T1); Overall 0.47. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Checkpoint PixArt-XL-2-1024-MS (Sec. 4.1). Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "PixArt-Alpha",
      "table": "S4.T1",
      "reported_value": "0.47",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "PixArt-Alpha",
        "0.45",
        "0.50",
        "0.48",
        "0.49",
        "0.56",
        "0.34",
        "0.47"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "playground-v2-5",
    "bench": "wise-wiscore-overall",
    "score": 0.49,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "playground-v2.5; Table 1 (S4.T1); Overall 0.49. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Checkpoint playground-v2.5-1024px-aesthetic (Sec. 4.1). Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "playground-v2.5",
      "table": "S4.T1",
      "reported_value": "0.49",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "playground-v2.5",
        "0.49",
        "0.58",
        "0.55",
        "0.43",
        "0.48",
        "0.33",
        "0.49"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "sd-1-5",
    "bench": "wise-wiscore-overall",
    "score": 0.32,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-v1-5; Table 1 (S4.T1); Overall 0.32. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Checkpoint stable-diffusion-v1-5. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-v1-5",
      "table": "S4.T1",
      "reported_value": "0.32",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-v1-5",
        "0.34",
        "0.35",
        "0.32",
        "0.28",
        "0.29",
        "0.21",
        "0.32"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "sd-2-1",
    "bench": "wise-wiscore-overall",
    "score": 0.32,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-2-1; Table 1 (S4.T1); Overall 0.32. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Checkpoint stable-diffusion-2-1. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-2-1",
      "table": "S4.T1",
      "reported_value": "0.32",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-2-1",
        "0.30",
        "0.38",
        "0.35",
        "0.33",
        "0.34",
        "0.21",
        "0.32"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "sd-3-medium",
    "bench": "wise-wiscore-overall",
    "score": 0.42,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-3-medium; Table 1 (S4.T1); Overall 0.42. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-3-medium",
      "table": "S4.T1",
      "reported_value": "0.42",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-3-medium",
        "0.42",
        "0.44",
        "0.48",
        "0.39",
        "0.47",
        "0.29",
        "0.42"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "sd-3-5-medium",
    "bench": "wise-wiscore-overall",
    "score": 0.45,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-3.5-medium; Table 1 (S4.T1); Overall 0.45. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-3.5-medium",
      "table": "S4.T1",
      "reported_value": "0.45",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-3.5-medium",
        "0.43",
        "0.50",
        "0.52",
        "0.41",
        "0.53",
        "0.33",
        "0.45"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "sd-3-5-large",
    "bench": "wise-wiscore-overall",
    "score": 0.46,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-3.5-large; Table 1 (S4.T1); Overall 0.46. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-3.5-large",
      "table": "S4.T1",
      "reported_value": "0.46",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-3.5-large",
        "0.44",
        "0.50",
        "0.58",
        "0.44",
        "0.52",
        "0.31",
        "0.46"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "bagel",
    "bench": "wise-wiscore-overall",
    "score": 0.7,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "BAGEL+CoT; Table 1 (S4.T1); Overall 0.70. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Setting: chain-of-thought (thinking) enabled, the model's highest reasoning setting; same table reports BAGEL without CoT at 0.52 (not recorded separately). Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "BAGEL+CoT",
      "table": "S4.T1",
      "reported_value": "0.70",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "BAGEL+CoT",
        "0.76",
        "0.69",
        "0.75",
        "0.65",
        "0.75",
        "0.58",
        "0.70"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "qwen-image",
    "bench": "wise-wiscore-overall",
    "score": 0.62,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "Qwen-Image; Table 1 (S4.T1); Overall 0.62. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "Qwen-Image",
      "table": "S4.T1",
      "reported_value": "0.62",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "Qwen-Image",
        "0.62",
        "0.63",
        "0.77",
        "0.57",
        "0.75",
        "0.40",
        "0.62"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "uniworld-v1",
    "bench": "wise-wiscore-overall",
    "score": 0.55,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "lab",
    "date": "2026-06-02",
    "note": "UniWorld-V1; Table 1 (S4.T1); Overall 0.55. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. UniWorld-V1 (arXiv 2506.03147) shares authors Bin Lin, Yuwei Niu and Li Yuan with WISE, so this row is a lab (own-group) result. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "UniWorld-V1",
      "table": "S4.T1",
      "reported_value": "0.55",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "UniWorld-V1",
        "0.53",
        "0.55",
        "0.73",
        "0.45",
        "0.59",
        "0.41",
        "0.55"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors overlapping with the model authors (PKU-YuanGroup)"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "metaquery-xl",
    "bench": "wise-wiscore-overall",
    "score": 0.55,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "MetaQuery-XL; Table 1 (S4.T1); Overall 0.55. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "MetaQuery-XL",
      "table": "S4.T1",
      "reported_value": "0.55",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "MetaQuery-XL",
        "0.56",
        "0.55",
        "0.62",
        "0.49",
        "0.63",
        "0.41",
        "0.55"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "blip3o-8b",
    "bench": "wise-wiscore-overall",
    "score": 0.52,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "BLIP3o-8B; Table 1 (S4.T1); Overall 0.52. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "BLIP3o-8B",
      "table": "S4.T1",
      "reported_value": "0.52",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "BLIP3o-8B",
        "0.49",
        "0.51",
        "0.63",
        "0.54",
        "0.63",
        "0.37",
        "0.52"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "emu3",
    "bench": "wise-wiscore-overall",
    "score": 0.39,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "Emu3; Table 1 (S4.T1); Overall 0.39. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "Emu3",
      "table": "S4.T1",
      "reported_value": "0.39",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "Emu3",
        "0.34",
        "0.45",
        "0.48",
        "0.41",
        "0.45",
        "0.27",
        "0.39"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "janus-pro-7b",
    "bench": "wise-wiscore-overall",
    "score": 0.35,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "Janus-Pro-7B; Table 1 (S4.T1); Overall 0.35. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "Janus-Pro-7B",
      "table": "S4.T1",
      "reported_value": "0.35",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "Janus-Pro-7B",
        "0.30",
        "0.37",
        "0.49",
        "0.36",
        "0.42",
        "0.26",
        "0.35"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "janus-pro-1b",
    "bench": "wise-wiscore-overall",
    "score": 0.26,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "Janus-Pro-1B; Table 1 (S4.T1); Overall 0.26. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "Janus-Pro-1B",
      "table": "S4.T1",
      "reported_value": "0.26",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "Janus-Pro-1B",
        "0.20",
        "0.28",
        "0.45",
        "0.24",
        "0.32",
        "0.16",
        "0.26"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "janus-1-3b",
    "bench": "wise-wiscore-overall",
    "score": 0.23,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "Janus-1.3B; Table 1 (S4.T1); Overall 0.23. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "Janus-1.3B",
      "table": "S4.T1",
      "reported_value": "0.23",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "Janus-1.3B",
        "0.16",
        "0.26",
        "0.35",
        "0.28",
        "0.30",
        "0.14",
        "0.23"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "vila-u-7b-256",
    "bench": "wise-wiscore-overall",
    "score": 0.31,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "vila-u-7b-256; Table 1 (S4.T1); Overall 0.31. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "vila-u-7b-256",
      "table": "S4.T1",
      "reported_value": "0.31",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "vila-u-7b-256",
        "0.26",
        "0.33",
        "0.37",
        "0.35",
        "0.39",
        "0.23",
        "0.31"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1; no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "gemini-omni-flash",
    "bench": "sci-vbench-testmini-human",
    "score": 0.636,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Gemini-Omni-Flash; Table 4 Human Eval Avg. 3.18/5; testmini 150 prompts, expert 1–5 rubric ratings; API gemini-omni-flash-preview, 1280×720, 24 fps, 10 s; 6 prompts rejected by provider filter.",
    "evidence": {
      "source_model_label": "Gemini-Omni-Flash (Google, 2026)",
      "table": "S5.T4",
      "reported_value": "3.18",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Gemini-Omni-Flash (Google, 2026)",
        "2026-06",
        "–",
        "4.12",
        "2.53",
        "3.52",
        "3.34",
        "3.38",
        "3.76",
        "2.53",
        "3.37",
        "3.06",
        "3.18"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Gemini-Omni-Flash",
        "2026-06",
        "gemini-omni-flash-preview",
        "API",
        "1280",
        "720",
        "24",
        "10s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "happyhorse-1-1",
    "bench": "sci-vbench-testmini-human",
    "score": 0.612,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "HappyHorse-1.1; Table 4 Human Eval Avg. 3.06/5; testmini 150 prompts, expert 1–5 rubric ratings; API happyhorse-1.1-t2v, 1280×720, 24 fps, 15 s; provider watermark present.",
    "evidence": {
      "source_model_label": "HappyHorse-1.1 (Alibaba Cloud, 2026)",
      "table": "S5.T4",
      "reported_value": "3.06",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "HappyHorse-1.1 (Alibaba Cloud, 2026)",
        "2026-06",
        "–",
        "3.96",
        "2.66",
        "3.21",
        "2.76",
        "3.15",
        "3.75",
        "2.85",
        "3.09",
        "2.57",
        "3.06"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "HappyHorse-1.1",
        "2026-06",
        "happyhorse-1.1-t2v",
        "API",
        "1280",
        "720",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "seedance-2-0",
    "bench": "sci-vbench-testmini-human",
    "score": 0.588,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Seedance-2.0; Table 4 Human Eval Avg. 2.94/5; testmini 150 prompts, expert 1–5 rubric ratings; API doubao-seedance-2-0-260128, 1280×720, 24 fps, 15 s; 2 prompts rejected by provider filter.",
    "evidence": {
      "source_model_label": "Seedance-2.0 (ByteDance Seed, 2026)",
      "table": "S5.T4",
      "reported_value": "2.94",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Seedance-2.0 (ByteDance Seed, 2026)",
        "2026-02",
        "–",
        "3.86",
        "2.46",
        "3.17",
        "3.06",
        "3.14",
        "3.73",
        "2.32",
        "3.01",
        "2.68",
        "2.94"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "ByteDance Seed",
        "Seedance-2.0",
        "2026-02",
        "doubao-seedance-2-0-260128",
        "API",
        "1280",
        "720",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "veo-3-1",
    "bench": "sci-vbench-testmini-human",
    "score": 0.56,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Veo-3.1; Table 4 Human Eval Avg. 2.80/5; testmini 150 prompts, expert 1–5 rubric ratings; API veo-3.1-generate-preview, 1280×720, 24 fps, 8 s.",
    "evidence": {
      "source_model_label": "Veo-3.1 (Google DeepMind, 2025)",
      "table": "S5.T4",
      "reported_value": "2.80",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Veo-3.1 (Google DeepMind, 2025)",
        "2025-10",
        "–",
        "4.10",
        "2.26",
        "2.83",
        "2.75",
        "2.98",
        "3.53",
        "2.66",
        "2.65",
        "2.37",
        "2.80"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Google",
        "Veo-3.1",
        "2025-10",
        "veo-3.1-generate-preview",
        "API",
        "1280",
        "720",
        "24",
        "8s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "veo-3-1-fast",
    "bench": "sci-vbench-testmini-human",
    "score": 0.558,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Veo-3.1-Fast; Table 4 Human Eval Avg. 2.79/5; testmini 150 prompts, expert 1–5 rubric ratings; API veo-3.1-fast-generate-preview, 1280×720, 24 fps, 8 s.",
    "evidence": {
      "source_model_label": "Veo-3.1-Fast (Google DeepMind, 2025)",
      "table": "S5.T4",
      "reported_value": "2.79",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Veo-3.1-Fast (Google DeepMind, 2025)",
        "2025-10",
        "–",
        "4.03",
        "2.34",
        "2.92",
        "2.70",
        "3.00",
        "3.45",
        "2.71",
        "2.67",
        "2.34",
        "2.79"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Veo-3.1-Fast",
        "2025-10",
        "veo-3.1-fast-generate-preview",
        "API",
        "1280",
        "720",
        "24",
        "8s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "sora-2",
    "bench": "sci-vbench-testmini-human",
    "score": 0.536,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Sora-2; Table 4 Human Eval Avg. 2.68/5; testmini 150 prompts, expert 1–5 rubric ratings; API sora-2, 1280×720, 30 fps, 15 s.",
    "evidence": {
      "source_model_label": "Sora-2 (The Sora Team, 2025)",
      "table": "S5.T4",
      "reported_value": "2.68",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Sora-2 (The Sora Team, 2025)",
        "2025-09",
        "–",
        "3.82",
        "2.29",
        "3.12",
        "2.72",
        "2.98",
        "3.40",
        "2.29",
        "2.77",
        "2.24",
        "2.68"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "OpenAI",
        "Sora-2",
        "2025-09",
        "sora-2",
        "API",
        "1280",
        "720",
        "30",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "wan-2-6",
    "bench": "sci-vbench-testmini-human",
    "score": 0.504,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Wan-2.6; Table 4 Human Eval Avg. 2.52/5; testmini 150 prompts, expert 1–5 rubric ratings; API wan2.6-t2v, 1280×720, 30 fps, 15 s.",
    "evidence": {
      "source_model_label": "Wan-2.6 (Wan AI Team, 2025)",
      "table": "S5.T4",
      "reported_value": "2.52",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Wan-2.6 (Wan AI Team, 2025)",
        "2025-12",
        "–",
        "3.89",
        "2.61",
        "2.43",
        "1.79",
        "2.68",
        "3.42",
        "2.82",
        "2.27",
        "1.58",
        "2.52"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Alibaba",
        "Wan-2.6",
        "2025-12",
        "wan2.6-t2v",
        "API",
        "1280",
        "720",
        "30",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "minimax-h3",
    "bench": "sci-vbench-testmini-human",
    "score": 0.516,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "MiniMax-H3; Table 4 Human Eval Avg. 2.58/5; testmini 150 prompts, expert 1–5 rubric ratings; HF MiniMaxAI/MiniMax-H3, 1344×768, 24 fps, 15 s.",
    "evidence": {
      "source_model_label": "MiniMax-H3 (MiniMax, 2026)",
      "table": "S5.T4",
      "reported_value": "2.58",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "MiniMax-H3 (MiniMax, 2026)",
        "2026-08",
        "2.61",
        "3.94",
        "2.62",
        "2.52",
        "1.63",
        "2.68",
        "3.64",
        "2.83",
        "2.29",
        "1.54",
        "2.58"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "MiniMax",
        "MiniMax-H3",
        "2026-08",
        "MiniMaxAI/MiniMax-H3",
        "Open",
        "1344",
        "768",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "sci-vbench-testmini-human",
    "score": 0.468,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "HunyuanVideo-1.5; Table 4 Human Eval Avg. 2.34/5; testmini 150 prompts, expert 1–5 rubric ratings; HF tencent/HunyuanVideo-1.5, HunyuanVideo-1.5-480P-T2V (Section 5.1), 848×480, 10 fps, 15 s.",
    "evidence": {
      "source_model_label": "HunyuanVideo-1.5 (Tencent, 2025)",
      "table": "S5.T4",
      "reported_value": "2.34",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "HunyuanVideo-1.5 (Tencent, 2025)",
        "2025-11",
        "2.43",
        "3.79",
        "2.66",
        "1.87",
        "1.45",
        "2.44",
        "3.45",
        "2.90",
        "1.80",
        "1.21",
        "2.34"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Tencent",
        "HunyuanVideo-1.5",
        "2025-11",
        "tencent/HunyuanVideo-1.5",
        "Open",
        "848",
        "480",
        "10",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "longcat-video",
    "bench": "sci-vbench-testmini-human",
    "score": 0.454,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "LongCat-Video; Table 4 Human Eval Avg. 2.27/5; testmini 150 prompts, expert 1–5 rubric ratings; HF meituan-longcat/LongCat-Video, 832×480, 10 fps, 15 s.",
    "evidence": {
      "source_model_label": "LongCat-Video (Meituan LongCat Team et al., 2025)",
      "table": "S5.T4",
      "reported_value": "2.27",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "LongCat-Video (Meituan LongCat Team et al., 2025)",
        "2025-10",
        "2.34",
        "3.92",
        "2.46",
        "1.76",
        "1.24",
        "2.34",
        "3.45",
        "2.76",
        "1.75",
        "1.13",
        "2.27"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Meituan",
        "LongCat-Video",
        "2025-10",
        "meituan-longcat/LongCat-Video",
        "Open",
        "832",
        "480",
        "10",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "cosmos3-nano",
    "bench": "sci-vbench-testmini-human",
    "score": 0.446,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Cosmos3-Nano; Table 4 Human Eval Avg. 2.23/5; testmini 150 prompts, expert 1–5 rubric ratings; HF nvidia/Cosmos3-Nano used for T2V, 1280×720, 24 fps, 15 s.",
    "evidence": {
      "source_model_label": "Cosmos3-Nano (NVIDIA, 2026)",
      "table": "S5.T4",
      "reported_value": "2.23",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Cosmos3-Nano (NVIDIA, 2026)",
        "2026-06",
        "2.43",
        "4.05",
        "2.73",
        "1.76",
        "1.27",
        "2.45",
        "3.37",
        "2.73",
        "1.67",
        "1.14",
        "2.23"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "NVIDIA",
        "Cosmos3-Nano",
        "2026-06",
        "nvidia/Cosmos3-Nano",
        "Open",
        "1280",
        "720",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "sci-vbench-testmini-human",
    "score": 0.444,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Wan2.2-5B; Table 4 Human Eval Avg. 2.22/5; testmini 150 prompts, expert 1–5 rubric ratings; HF Wan-AI/Wan2.2-TI2V-5B (Wan2.2-5B-T2V in Section 5.1), native 1280×704, 16 fps, 15 s.",
    "evidence": {
      "source_model_label": "Wan2.2-5B (Team Wan et al., 2025)",
      "table": "S5.T4",
      "reported_value": "2.22",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "Wan2.2-5B (Team Wan et al., 2025)",
        "2025-07",
        "2.43",
        "3.87",
        "2.79",
        "1.85",
        "1.33",
        "2.46",
        "3.05",
        "2.82",
        "1.79",
        "1.21",
        "2.22"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Alibaba",
        "Wan2.2-5B",
        "2025-07",
        "Wan-AI/Wan2.2-TI2V-5B",
        "Open",
        "1280",
        "704",
        "16",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "ltx-2-19b-distilled",
    "bench": "sci-vbench-testmini-human",
    "score": 0.424,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "LTX-2; Table 4 Human Eval Avg. 2.12/5; testmini 150 prompts, expert 1–5 rubric ratings; HF Lightricks/LTX-2 (19B-distilled) = LTX-2.0-19B-distilled in Section 5.1, 1536×1024, 24 fps, 15 s; distilled checkpoint kept separate from ltx-2.",
    "evidence": {
      "source_model_label": "LTX-2 (HaCohen et al., 2026)",
      "table": "S5.T4",
      "reported_value": "2.12",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "LTX-2 (HaCohen et al., 2026)",
        "2026-01",
        "2.31",
        "4.07",
        "2.36",
        "1.51",
        "1.26",
        "2.30",
        "3.13",
        "2.74",
        "1.48",
        "1.12",
        "2.12"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Lightricks",
        "LTX-2",
        "2026-01",
        "Lightricks/LTX-2 (19B-distilled)",
        "Open",
        "1536",
        "1024",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "sci-vbench-testmini-human",
    "score": 0.386,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "CogVideoX1.5-5B; Table 4 Human Eval Avg. 1.93/5; testmini 150 prompts, expert 1–5 rubric ratings; HF THUDM/CogVideoX1.5-5B, 1360×768, 16 fps, 5 s.",
    "evidence": {
      "source_model_label": "CogVideoX1.5-5B (Yang et al., 2025)",
      "table": "S5.T4",
      "reported_value": "1.93",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "CogVideoX1.5-5B (Yang et al., 2025)",
        "2024-11",
        "2.30",
        "3.98",
        "2.11",
        "1.92",
        "1.40",
        "2.35",
        "2.68",
        "2.11",
        "1.75",
        "1.19",
        "1.93"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "Zhipu AI",
        "CogVideoX1.5-5B",
        "2024-11",
        "THUDM/CogVideoX1.5-5B",
        "Open",
        "1360",
        "768",
        "16",
        "5s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "ltx-2-3",
    "bench": "sci-vbench-testmini-human",
    "score": 0.384,
    "source": "https://arxiv.org/html/2608.09873v1#S5.T4",
    "source_id": "visual-scivbench",
    "by": "independent",
    "date": "2026-08-10",
    "note": "LTX-2.3; Table 4 Human Eval Avg. 1.92/5; testmini 150 prompts, expert 1–5 rubric ratings; HF Lightricks/LTX-2.3, 1536×1024, 24 fps, 15 s.",
    "evidence": {
      "source_model_label": "LTX-2.3 (HaCohen et al., 2026)",
      "table": "S5.T4",
      "reported_value": "1.92",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author generation and expert rating",
      "protocol": "Verbatim prompts, default configuration, testmini split (150 examples); expert annotators assign 1–5 integer scores per dimension; Human Eval Avg. = unweighted mean of LPF, SC, PG, SCC.",
      "source_row": [
        "LTX-2.3 (HaCohen et al., 2026)",
        "2026-03",
        "2.27",
        "3.96",
        "2.16",
        "1.80",
        "1.28",
        "2.30",
        "2.66",
        "2.15",
        "1.70",
        "1.18",
        "1.92"
      ],
      "source_header": [
        "Models",
        "Release",
        "Full Avg.",
        "VT",
        "SC",
        "PG",
        "SCC",
        "Avg.",
        "LPF",
        "SC",
        "PG",
        "SCC",
        "Avg."
      ],
      "configuration_row": [
        "LTX-2.3",
        "2026-03",
        "Lightricks/LTX-2.3",
        "Open",
        "1536",
        "1024",
        "24",
        "15s"
      ],
      "date_basis": "Report publication/version date (v1, 2026-08-10), not an asserted generation date",
      "model_identity_scope": "Exact API model id or HF repository from Table 6; no undisclosed snapshot claimed beyond the stated id.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.09873v1#S5.SS1.p1.1",
      "provenance_quote": "Every testmini video is rated by an expert, and Table 4 reports those ratings for all 16 models. (S4.SS3) … All videos are generated from the verbatim benchmark prompts under each model’s default configuration (Table 6 caption).",
      "evaluator_relationship": "academic benchmark authors (Tongji, Yale, Zhejiang, UCAS) evaluating other providers' models"
    },
    "evaluator": "Sci-VBench authors (Tongji University / Yale University / Zhejiang University / UCAS)",
    "source_group": "visual-scivbench:sci-vbench-testmini-human"
  },
  {
    "model": "seedream-4-0",
    "bench": "genexam-relaxed",
    "score": 0.532,
    "source": "https://arxiv.org/html/2509.14232v5#S3.T2",
    "source_id": "visual-genexam",
    "by": "independent",
    "date": "2026-05-14",
    "note": "Seedream 4.0; Table 2 Overall relaxed 53.2% (strict 7.8%); default T2I configuration; GPT-5 (low) judge; row first appears in v5 (absent from v4 Table 2), with no citation, so it is the authors' own v5 run; the Section 4.1 model inventory text was not updated to list it. Reported release label retained; API quality tier/snapshot not specified.",
    "evidence": {
      "source_model_label": "Seedream 4.0",
      "table": "S3.T2",
      "reported_value": "53.2",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "1,000 prompts; default T2I configuration per model; GPT-5 (reasoning effort low) scores per-sample scoring points and visual plausibility; relaxed = 0.7 semantic + 0.1 spelling + 0.1 logic + 0.1 readability.",
      "source_row": [
        "Seedream 4.0",
        "2.6",
        "39.8",
        "3.5",
        "49.0",
        "5.9",
        "46.1",
        "18.6",
        "71.0",
        "10.6",
        "65.1",
        "6.9",
        "52.2",
        "11.7",
        "60.0",
        "5.2",
        "56.0",
        "0.0",
        "34.5",
        "7.3",
        "56.7",
        "7.8",
        "53.2"
      ],
      "source_header": [
        "Model",
        "Math Str",
        "Math Rel",
        "Phy Str",
        "Phy Rel",
        "Chem Str",
        "Chem Rel",
        "Bio Str",
        "Bio Rel",
        "Geo Str",
        "Geo Rel",
        "Comp Str",
        "Comp Rel",
        "Eng Str",
        "Eng Rel",
        "Econ Str",
        "Econ Rel",
        "Music Str",
        "Music Rel",
        "Hist Str",
        "Hist Rel",
        "Overall Str",
        "Overall Rel"
      ],
      "date_basis": "Report version date (v5, 2026-05-14), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2509.14232v5#S4.SS1.p1.1",
      "provenance_quote": "We select 17 representative models for comparison ... We use the default T2I configuration for each of the models for inference. (S4.SS1) We use GPT-5 as the evaluator model and set the reasoning effort as 'low'. (S4)",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "GenExam authors (Shanghai AI Laboratory / SJTU / Tsinghua / CUHK)",
    "source_group": "visual-genexam:genexam-relaxed"
  },
  {
    "model": "gpt-image-2",
    "bench": "fepbench-if",
    "score": 0.664,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "GPT Image 2; Table 2 IF mean 0.664 (RE 0.334, SP 0.703 not imported); API, 1024×1024/1536×1024/1024×1536; tested April 2026.",
    "evidence": {
      "source_model_label": "GPT Image 2",
      "table": "S4.T2",
      "reported_value": "0.664",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "GPT Image 2",
        "0.809",
        "0.389",
        "0.573",
        "0.886",
        "0.417",
        "0.141",
        "0.443",
        "0.774",
        "0.479",
        "0.856",
        "0.664",
        "0.334",
        "0.703"
      ],
      "configuration_row": [
        "GPT-image-2",
        "API",
        "1024×\\times1024 / 1536×\\times1024 / 1024×\\times1536",
        "Not exposed",
        "Not exposed",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "nano-banana-pro",
    "bench": "fepbench-if",
    "score": 0.629,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "Nano Banana Pro; Table 2 IF mean 0.629 (RE 0.281, SP 0.733 not imported); API, 1920×1920/1920×1080/1080×1920; tested April 2026.",
    "evidence": {
      "source_model_label": "Nano Banana Pro",
      "table": "S4.T2",
      "reported_value": "0.629",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "Nano Banana Pro",
        "0.772",
        "0.333",
        "0.541",
        "0.870",
        "0.329",
        "0.119",
        "0.395",
        "0.812",
        "0.507",
        "0.879",
        "0.629",
        "0.281",
        "0.733"
      ],
      "configuration_row": [
        "Nano Banana Pro",
        "API",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "Not exposed",
        "Not exposed",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "fepbench-if",
    "score": 0.598,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "GPT Image 1.5; Table 2 IF mean 0.598 (RE 0.294, SP 0.693 not imported); API, 1024×1024/1536×1024/1024×1536; tested April 2026.",
    "evidence": {
      "source_model_label": "GPT Image 1.5",
      "table": "S4.T2",
      "reported_value": "0.598",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "GPT Image 1.5",
        "0.793",
        "0.361",
        "0.529",
        "0.710",
        "0.344",
        "0.099",
        "0.438",
        "0.859",
        "0.438",
        "0.784",
        "0.598",
        "0.294",
        "0.693"
      ],
      "configuration_row": [
        "GPT-image-1.5",
        "API",
        "1024×\\times1024 / 1536×\\times1024 / 1024×\\times1536",
        "Not exposed",
        "Not exposed",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "qwen-image-2-0-pro",
    "bench": "fepbench-if",
    "score": 0.515,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "Qwen-Image-2.0 Pro; Table 2 IF mean 0.515 (RE 0.234, SP 0.747 not imported); API, 1920×1920/1920×1080/1080×1920; tested April 2026.",
    "evidence": {
      "source_model_label": "Qwen-Image-2.0 Pro",
      "table": "S4.T2",
      "reported_value": "0.515",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "Qwen-Image-2.0 Pro",
        "0.751",
        "0.234",
        "0.432",
        "0.641",
        "0.277",
        "0.054",
        "0.371",
        "0.852",
        "0.565",
        "0.824",
        "0.515",
        "0.234",
        "0.747"
      ],
      "configuration_row": [
        "Qwen-Image-2.0 Pro",
        "API",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "Not exposed",
        "Not exposed",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "flux-2-dev",
    "bench": "fepbench-if",
    "score": 0.531,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "FLUX.2 [dev]; Table 2 IF mean 0.531 (RE 0.244, SP 0.655 not imported); open weights, 50 steps, guidance 5.0; tested April 2026.",
    "evidence": {
      "source_model_label": "FLUX.2 [dev]",
      "table": "S4.T2",
      "reported_value": "0.531",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "FLUX.2 [dev]",
        "0.724",
        "0.289",
        "0.424",
        "0.686",
        "0.306",
        "0.058",
        "0.368",
        "0.832",
        "0.320",
        "0.812",
        "0.531",
        "0.244",
        "0.655"
      ],
      "configuration_row": [
        "FLUX.2 [dev]",
        "Open-source",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "50",
        "5.0",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "fepbench-if",
    "score": 0.41,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "HunyuanImage-3.0; Table 2 IF mean 0.410 (RE 0.200, SP 0.613 not imported); open weights, 50 steps, guidance 5.0, 1024×1024/1536×1024/1024×1536; tested April 2026.",
    "evidence": {
      "source_model_label": "HunyuanImage-3.0",
      "table": "S4.T2",
      "reported_value": "0.410",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "HunyuanImage-3.0",
        "0.632",
        "0.087",
        "0.265",
        "0.655",
        "0.293",
        "0.029",
        "0.279",
        "0.746",
        "0.295",
        "0.798",
        "0.410",
        "0.200",
        "0.613"
      ],
      "configuration_row": [
        "HunyuanImage-3.0",
        "Open-source",
        "1024×\\times1024 / 1536×\\times1024 / 1024×\\times1536",
        "50",
        "5.0",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "z-image-turbo",
    "bench": "fepbench-if",
    "score": 0.39,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "Z-Image-Turbo; Table 2 IF mean 0.390 (RE 0.165, SP 0.606 not imported); open weights run at 50 steps and guidance 5.0 (not the model's few-step default); tested April 2026.",
    "evidence": {
      "source_model_label": "Z-Image-Turbo",
      "table": "S4.T2",
      "reported_value": "0.390",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "Z-Image-Turbo",
        "0.560",
        "0.179",
        "0.259",
        "0.562",
        "0.232",
        "0.030",
        "0.234",
        "0.791",
        "0.298",
        "0.730",
        "0.390",
        "0.165",
        "0.606"
      ],
      "configuration_row": [
        "Z-Image-Turbo",
        "Open-source",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "50",
        "5.0",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "edit-compass-en-overall",
    "score": 0.386,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "FLUX.1 Kontext Dev; S4.T3 English-instruction Overall 1.93 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.93/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "FLUX.1 Kontext Dev Labs et al. (2025)",
      "table": "S4.T3",
      "reported_value": "1.93",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "FLUX.1 Kontext Dev Labs et al. (2025)",
        "✗",
        "3.53",
        "2.98",
        "3.09",
        "2.41",
        "2.93",
        "2.81",
        "1.65",
        "3.22",
        "3.32",
        "1.27",
        "3.98",
        "4.78",
        "-",
        "-",
        "-",
        "2.41",
        "2.32",
        "2.63",
        "1.93"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "flux-2-dev",
    "bench": "edit-compass-en-overall",
    "score": 0.522,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "FLUX.2 Dev; S4.T3 English-instruction Overall 2.61 on the 1–5 Gemini-3.1-Pro judge scale, stored as 2.61/5. Open model; default settings not further specified.",
    "evidence": {
      "source_model_label": "FLUX.2 Dev Labs (2025)",
      "table": "S4.T3",
      "reported_value": "2.61",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "FLUX.2 Dev Labs (2025)",
        "✓\\checkmark",
        "4.24",
        "4.16",
        "3.96",
        "3.23",
        "4.36",
        "3.36",
        "2.05",
        "3.85",
        "3.71",
        "1.42",
        "2.85",
        "4.56",
        "3.16",
        "4.13",
        "3.14",
        "2.62",
        "4.32",
        "3.54",
        "2.61"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "omnigen2",
    "bench": "edit-compass-en-overall",
    "score": 0.376,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "OmniGen2; S4.T3 English-instruction Overall 1.88 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.88/5. Open model; default settings not further specified.",
    "evidence": {
      "source_model_label": "OmniGen2 Wu et al. (2025b)",
      "table": "S4.T3",
      "reported_value": "1.88",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "OmniGen2 Wu et al. (2025b)",
        "✓\\checkmark",
        "3.33",
        "3.34",
        "3.12",
        "2.47",
        "3.70",
        "3.19",
        "1.53",
        "3.33",
        "3.55",
        "1.11",
        "2.51",
        "3.69",
        "2.55",
        "1.77",
        "2.65",
        "2.27",
        "2.73",
        "2.73",
        "1.88"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "bagel",
    "bench": "edit-compass-en-overall",
    "score": 0.416,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Bagel-Think (thinking mode); S4.T3 English-instruction Overall 2.08 on the 1–5 Gemini-3.1-Pro judge scale, stored as 2.08/5. Thinking mode chosen over the non-thinking row (2.10) for consistency with the other BAGEL sources (highest reasoning setting).",
    "evidence": {
      "source_model_label": "Bagel-Think Deng et al. (2025)",
      "table": "S4.T3",
      "reported_value": "2.08",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Bagel-Think Deng et al. (2025)",
        "✓\\checkmark",
        "3.52",
        "3.92",
        "3.14",
        "2.55",
        "4.04",
        "3.15",
        "2.22",
        "3.09",
        "3.27",
        "1.21",
        "3.25",
        "3.48",
        "1.85",
        "2.01",
        "3.03",
        "2.19",
        "3.35",
        "2.89",
        "2.08"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "edit-compass-en-overall",
    "score": 0.516,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Step1X-Edit-v1p2; S4.T3 English-instruction Overall 2.58 on the 1–5 Gemini-3.1-Pro judge scale, stored as 2.58/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "Step1X-Edit-v1p2 Liu et al. (2025b)",
      "table": "S4.T3",
      "reported_value": "2.58",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Step1X-Edit-v1p2 Liu et al. (2025b)",
        "✗",
        "4.26",
        "4.31",
        "3.64",
        "3.16",
        "4.15",
        "3.12",
        "2.30",
        "4.09",
        "3.49",
        "1.26",
        "2.72",
        "4.41",
        "-",
        "-",
        "-",
        "2.94",
        "3.33",
        "2.81",
        "2.58"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "longcat-image-edit",
    "bench": "edit-compass-en-overall",
    "score": 0.53,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Longcat-Image-Edit; S4.T3 English-instruction Overall 2.65 on the 1–5 Gemini-3.1-Pro judge scale, stored as 2.65/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "Longcat-Image-Edit Team et al. (2025b)",
      "table": "S4.T3",
      "reported_value": "2.65",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Longcat-Image-Edit Team et al. (2025b)",
        "✗",
        "4.51",
        "4.48",
        "3.90",
        "3.54",
        "4.14",
        "3.38",
        "1.98",
        "3.94",
        "3.49",
        "1.31",
        "3.26",
        "4.21",
        "-",
        "-",
        "-",
        "2.72",
        "3.31",
        "3.19",
        "2.65"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "edit-compass-en-overall",
    "score": 0.538,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Qwen-Image-Edit-2511; S4.T3 English-instruction Overall 2.69 on the 1–5 Gemini-3.1-Pro judge scale, stored as 2.69/5. Open model; default settings not further specified.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit-2511 Wu et al. (2025a)",
      "table": "S4.T3",
      "reported_value": "2.69",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Qwen-Image-Edit-2511 Wu et al. (2025a)",
        "✓\\checkmark",
        "4.61",
        "4.39",
        "3.75",
        "3.81",
        "3.93",
        "3.16",
        "2.33",
        "3.56",
        "3.25",
        "1.26",
        "2.53",
        "4.09",
        "3.27",
        "3.55",
        "2.92",
        "2.80",
        "3.81",
        "3.02",
        "2.69"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "seedream-4-5",
    "bench": "edit-compass-en-overall",
    "score": 0.644,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Seedream 4.5; S4.T3 English-instruction Overall 3.22 on the 1–5 Gemini-3.1-Pro judge scale, stored as 3.22/5. Closed model evaluated via its official API.",
    "evidence": {
      "source_model_label": "Seedream 4.5 Seedream et al. (2025)",
      "table": "S4.T3",
      "reported_value": "3.22",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Seedream 4.5 Seedream et al. (2025)",
        "✓\\checkmark",
        "4.66",
        "4.36",
        "4.13",
        "4.36",
        "4.15",
        "3.89",
        "3.58",
        "4.07",
        "4.04",
        "1.60",
        "2.90",
        "4.50",
        "4.34",
        "4.13",
        "3.42",
        "4.04",
        "4.11",
        "3.44",
        "3.22"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "wan-2-7-image",
    "bench": "edit-compass-en-overall",
    "score": 0.646,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Wan2.7-image; S4.T3 English-instruction Overall 3.23 on the 1–5 Gemini-3.1-Pro judge scale, stored as 3.23/5. Closed model evaluated via its official API.",
    "evidence": {
      "source_model_label": "Wan2.7-image Wan (2025)",
      "table": "S4.T3",
      "reported_value": "3.23",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Wan2.7-image Wan (2025)",
        "✓\\checkmark",
        "4.60",
        "4.36",
        "4.07",
        "4.40",
        "4.11",
        "3.92",
        "3.65",
        "4.16",
        "4.01",
        "1.61",
        "2.81",
        "4.49",
        "4.25",
        "4.23",
        "3.41",
        "3.97",
        "4.04",
        "3.41",
        "3.23"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "nano-banana-2",
    "bench": "edit-compass-en-overall",
    "score": 0.748,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Nano Banana 2; S4.T3 English-instruction Overall 3.74 on the 1–5 Gemini-3.1-Pro judge scale, stored as 3.74/5. Closed model evaluated via its official API.",
    "evidence": {
      "source_model_label": "Nano Banana 2 Google (2026a)",
      "table": "S4.T3",
      "reported_value": "3.74",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Nano Banana 2 Google (2026a)",
        "✓\\checkmark",
        "4.79",
        "4.54",
        "4.14",
        "4.50",
        "4.33",
        "3.71",
        "4.20",
        "4.35",
        "4.19",
        "2.77",
        "4.03",
        "4.63",
        "4.50",
        "4.37",
        "3.41",
        "4.37",
        "4.46",
        "3.50",
        "3.74"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "nano-banana-pro",
    "bench": "edit-compass-en-overall",
    "score": 0.798,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Nano Banana Pro; S4.T3 English-instruction Overall 3.99 on the 1–5 Gemini-3.1-Pro judge scale, stored as 3.99/5. Closed model evaluated via its official API.",
    "evidence": {
      "source_model_label": "Nano Banana Pro Google DeepMind (2025)",
      "table": "S4.T3",
      "reported_value": "3.99",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Nano Banana Pro Google DeepMind (2025)",
        "✓\\checkmark",
        "4.76",
        "4.70",
        "4.11",
        "4.54",
        "4.58",
        "3.79",
        "4.33",
        "4.49",
        "4.28",
        "3.61",
        "4.25",
        "4.73",
        "4.43",
        "4.29",
        "3.44",
        "4.28",
        "4.40",
        "3.53",
        "3.99"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "nano-banana-pro",
    "bench": "geditbench-v2-overall-elo",
    "score": 1096,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Nano Banana Pro (26-03-04); S5.T3 Overall Elo 1,096 (95% CI -6/+6); 1,156 valid samples of 1,200. Closed model accessed on 2026-03-04 (label date).",
    "evidence": {
      "source_model_label": "Nano Banana Pro (26-03-04)",
      "table": "S5.T3",
      "reported_value": "1,096",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Nano Banana Pro (26-03-04)",
        "1,156",
        "1,126",
        "-13/+15",
        "1,066",
        "-9/+10",
        "1,108",
        "-11/+11",
        "1,096",
        "-6/+6",
        "1,251",
        "2"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,156"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "seedream-4-5",
    "bench": "geditbench-v2-overall-elo",
    "score": 1089,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Seedream 4.5 (26-03-11); S5.T3 Overall Elo 1,089 (95% CI -7/+7); 1,190 valid samples of 1,200. Closed model accessed on 2026-03-11 (label date).",
    "evidence": {
      "source_model_label": "Seedream 4.5 (26-03-11)",
      "table": "S5.T3",
      "reported_value": "1,089",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Seedream 4.5 (26-03-11)",
        "1,190",
        "1,111",
        "-12/+12",
        "1,142",
        "-11/+11",
        "1,030",
        "-11/+12",
        "1,089",
        "-7/+7",
        "1,196",
        "3"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-7/+7",
      "samples": "1,190"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "geditbench-v2-overall-elo",
    "score": 1071,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "GPT Image 1.5 (26-03-04); S5.T3 Overall Elo 1,071 (95% CI -7/+6); 1,081 valid samples of 1,200. Closed model accessed on 2026-03-04 (label date).",
    "evidence": {
      "source_model_label": "GPT Image 1.5 (26-03-04)",
      "table": "S5.T3",
      "reported_value": "1,071",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet GPT Image 1.5 (26-03-04)",
        "1,081",
        "1,260",
        "-13/+15",
        "1,149",
        "-12/+12",
        "846",
        "-13/+13",
        "1,071",
        "-7/+6",
        "1,270",
        "1"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-7/+6",
      "samples": "1,081"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "geditbench-v2-overall-elo",
    "score": 1039,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "FLUX.2 [klein] 9B; S5.T3 Overall Elo 1,039 (95% CI -6/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "FLUX.2 [klein] 9B",
      "table": "S5.T3",
      "reported_value": "1,039",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet FLUX.2 [klein] 9B",
        "1,200",
        "1,083",
        "-13/+12",
        "1,025",
        "-11/+10",
        "1,019",
        "-10/+9",
        "1,039",
        "-6/+6",
        "1,166",
        "4"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "geditbench-v2-overall-elo",
    "score": 1038,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Qwen-Image-Edit-2511; S5.T3 Overall Elo 1,038 (95% CI -6/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit-2511",
      "table": "S5.T3",
      "reported_value": "1,038",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Qwen-Image-Edit-2511",
        "1,200",
        "1,095",
        "-10/+10",
        "1,060",
        "-11/+11",
        "972",
        "-9/+10",
        "1,038",
        "-6/+6",
        "1,164",
        "5"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "flux-2-klein-4b",
    "bench": "geditbench-v2-overall-elo",
    "score": 1031,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "FLUX.2 [klein] 4B; S5.T3 Overall Elo 1,031 (95% CI -6/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "FLUX.2 [klein] 4B",
      "table": "S5.T3",
      "reported_value": "1,031",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet FLUX.2 [klein] 4B",
        "1,200",
        "1,007",
        "-12/+12",
        "1,019",
        "-10/+10",
        "1,070",
        "-10/+10",
        "1,031",
        "-6/+6",
        "1,107",
        "10"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "geditbench-v2-overall-elo",
    "score": 1014,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Qwen-Image-Edit-2509; S5.T3 Overall Elo 1,014 (95% CI -5/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit-2509",
      "table": "S5.T3",
      "reported_value": "1,014",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Qwen-Image-Edit-2509",
        "1,200",
        "1,033",
        "-10/+11",
        "1,062",
        "-10/+12",
        "955",
        "-9/+9",
        "1,014",
        "-5/+6",
        "1,142",
        "7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-5/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "qwen-image-edit",
    "bench": "geditbench-v2-overall-elo",
    "score": 1010,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Qwen-Image-Edit; S5.T3 Overall Elo 1,010 (95% CI -6/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit",
      "table": "S5.T3",
      "reported_value": "1,010",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Qwen-Image-Edit",
        "1,200",
        "991",
        "-10/+10",
        "1,073",
        "-11/+12",
        "971",
        "-11/+11",
        "1,010",
        "-6/+6",
        "1,088",
        "12"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "flux-2-dev",
    "bench": "geditbench-v2-overall-elo",
    "score": 1006,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "FLUX.2 [dev]; S5.T3 Overall Elo 1,006 (95% CI -7/+7); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "FLUX.2 [dev]",
      "table": "S5.T3",
      "reported_value": "1,006",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet FLUX.2 [dev]",
        "1,200",
        "1,037",
        "-12/+13",
        "965",
        "-10/+10",
        "1,018",
        "-11/+11",
        "1,006",
        "-7/+7",
        "1,137",
        "8"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-7/+7",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "longcat-image-edit",
    "bench": "geditbench-v2-overall-elo",
    "score": 1001,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "LongCat-Image-Edit; S5.T3 Overall Elo 1,001 (95% CI -6/+5); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "LongCat-Image-Edit",
      "table": "S5.T3",
      "reported_value": "1,001",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet LongCat-Image-Edit",
        "1,200",
        "1,018",
        "-10/+11",
        "968",
        "-10/+9",
        "1,017",
        "-10/+9",
        "1,001",
        "-6/+5",
        "1,111",
        "9"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+5",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "geditbench-v2-overall-elo",
    "score": 996,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "lab",
    "date": "2026-03-30",
    "note": "Step1X-Edit-v1p2; S5.T3 Overall Elo 996 (95% CI -6/+7); 1,200 valid samples of 1,200. Open model, default hyperparameters. StepFun co-authored the benchmark: lab row.",
    "evidence": {
      "source_model_label": "Step1X-Edit-v1p2",
      "table": "S5.T3",
      "reported_value": "996",
      "divisor": 1,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Step1X-Edit-v1p2",
        "1,200",
        "909",
        "-12/+12",
        "1,007",
        "-12/+11",
        "1,067",
        "-11/+11",
        "996",
        "-6/+7",
        "1,093",
        "11"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "model author evaluating own model (StepFun co-authors)",
      "ci95": "-6/+7",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "glm-image",
    "bench": "geditbench-v2-overall-elo",
    "score": 979,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "GLM-Image; S5.T3 Overall Elo 979 (95% CI -6/+6); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "S5.T3",
      "reported_value": "979",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet GLM-Image",
        "1,200",
        "787",
        "-13/+14",
        "1,023",
        "-11/+11",
        "1,109",
        "-13/+14",
        "979",
        "-6/+6",
        "930",
        "14"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-6/+6",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "omnigen2",
    "bench": "geditbench-v2-overall-elo",
    "score": 888,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "OmniGen V2; S5.T3 Overall Elo 888 (95% CI -7/+7); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "OmniGen V2",
      "table": "S5.T3",
      "reported_value": "888",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet OmniGen V2",
        "1,200",
        "807",
        "-13/+12",
        "910",
        "-12/+12",
        "929",
        "-13/+13",
        "888",
        "-7/+7",
        "919",
        "15"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-7/+7",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "geditbench-v2-overall-elo",
    "score": 869,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "FLUX.1 Kontext [dev]; S5.T3 Overall Elo 869 (95% CI -7/+8); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "FLUX.1 Kontext [dev]",
      "table": "S5.T3",
      "reported_value": "869",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet FLUX.1 Kontext [dev]",
        "1,200",
        "849",
        "-13/+13",
        "900",
        "-13/+14",
        "840",
        "-14/+13",
        "869",
        "-7/+8",
        "1,017",
        "13"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-7/+8",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "bagel",
    "bench": "geditbench-v2-overall-elo",
    "score": 851,
    "source": "https://arxiv.org/html/2603.28547v1#S5.T3",
    "source_id": "visual-geditbench-v2",
    "by": "independent",
    "date": "2026-03-30",
    "note": "Bagel; S5.T3 Overall Elo 851 (95% CI -8/+8); 1,200 valid samples of 1,200. Open model, default hyperparameters.",
    "evidence": {
      "source_model_label": "Bagel",
      "table": "S5.T3",
      "reported_value": "851",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GEditBench v2 1,200 queries; pairwise judgements (GPT-4o for IF/VQ, PVC-Judge for VC); shared Bradley–Terry fit to Elo; 1,000 bootstrap CI.",
      "source_row": [
        "∙\\bullet Bagel",
        "1,200",
        "820",
        "-13/+13",
        "694",
        "-17/+16",
        "987",
        "-13/+14",
        "851",
        "-8/+8",
        "915",
        "16"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.28547v1#S5.SS2.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "ci95": "-8/+8",
      "samples": "1,200"
    },
    "evaluator": "GEditBench v2 authors (NTU / StepFun / Southeast University)",
    "source_group": "visual-geditbench-v2:geditbench-v2-overall-elo"
  },
  {
    "model": "omnigen2",
    "bench": "unicbench-en-overall",
    "score": 0.61246,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "OmniGen2 [42]; S6.T4 Overall-EN 6.1246 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 6.1246/10. Open-source model; settings not tabulated.",
    "evidence": {
      "source_model_label": "OmniGen2 [42]",
      "table": "S6.T4",
      "reported_value": "6.1246",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "OmniGen2 [42]",
        "6.2455",
        "7.4973",
        "6.4891",
        "5.1240",
        "6.1246",
        "-",
        "-",
        "-",
        "-",
        "-"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "unicbench-en-overall",
    "score": 0.68045,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "FLUX.1-Kontext [3]; S6.T4 Overall-EN 6.8045 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 6.8045/10. Open-source model; settings not tabulated. Tier inferred as [dev] from the table's open-source grouping.",
    "evidence": {
      "source_model_label": "FLUX.1-Kontext [3]",
      "table": "S6.T4",
      "reported_value": "6.8045",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "FLUX.1-Kontext [3]",
        "6.7755",
        "8.4718",
        "7.3600",
        "5.5040",
        "6.8045",
        "-",
        "-",
        "-",
        "-",
        "-"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "bagel",
    "bench": "unicbench-en-overall",
    "score": 0.69794,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "BAGEL [8]; S6.T4 Overall-EN 6.9794 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 6.9794/10. Open-source model; settings not tabulated.",
    "evidence": {
      "source_model_label": "BAGEL [8]",
      "table": "S6.T4",
      "reported_value": "6.9794",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "BAGEL [8]",
        "7.2491",
        "8.1982",
        "7.1391",
        "5.2600",
        "6.9794",
        "7.3018",
        "8.2845",
        "7.3118",
        "5.2840",
        "7.1056"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "step1x-edit-v1-1",
    "bench": "unicbench-en-overall",
    "score": 0.69202,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Step1X-Edit-v1.1 [23]; S6.T4 Overall-EN 6.9202 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 6.9202/10. Open-source model; settings not tabulated.",
    "evidence": {
      "source_model_label": "Step1X-Edit-v1.1 [23]",
      "table": "S6.T4",
      "reported_value": "6.9202",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "Step1X-Edit-v1.1 [23]",
        "6.9945",
        "8.2045",
        "7.3382",
        "5.0400",
        "6.9202",
        "7.0282",
        "8.4118",
        "7.5600",
        "5.0560",
        "7.0620"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "qwen-image-edit",
    "bench": "unicbench-en-overall",
    "score": 0.77273,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Qwen-Image-Edit [41]; S6.T4 Overall-EN 7.7273 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 7.7273/10. Open-source model; settings not tabulated.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit [41]",
      "table": "S6.T4",
      "reported_value": "7.7273",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "Qwen-Image-Edit [41]",
        "8.2055",
        "8.0264",
        "8.0745",
        "6.4480",
        "7.7273",
        "8.3718",
        "7.8000",
        "8.2118",
        "6.6560",
        "7.7790"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "nano-banana",
    "bench": "unicbench-en-overall",
    "score": 0.78792,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Nano Banana [7]; S6.T4 Overall-EN 7.8792 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 7.8792/10. Closed model. Nano Banana cites the Gemini 2.5 report [7].",
    "evidence": {
      "source_model_label": "Nano Banana [7]",
      "table": "S6.T4",
      "reported_value": "7.8792",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "Nano Banana [7]",
        "7.9753",
        "8.9808",
        "8.1954",
        "6.8680",
        "7.8792",
        "8.1550",
        "9.0438",
        "8.3291",
        "6.8960",
        "8.0358"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "seededit-3-0",
    "bench": "unicbench-en-overall",
    "score": 0.78671,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "SeedEdit 3.0 [35]; S6.T4 Overall-EN 7.8671 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 7.8671/10. Closed model.",
    "evidence": {
      "source_model_label": "SeedEdit 3.0 [35]",
      "table": "S6.T4",
      "reported_value": "7.8671",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "SeedEdit 3.0 [35]",
        "8.2717",
        "8.4251",
        "7.8392",
        "6.9393",
        "7.8671",
        "8.3721",
        "8.4502",
        "7.9795",
        "6.8395",
        "7.9753"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "seedream-4-0",
    "bench": "unicbench-en-overall",
    "score": 0.80428,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Seedream 4.0 [31]; S6.T4 Overall-EN 8.0428 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 8.0428/10. Closed model.",
    "evidence": {
      "source_model_label": "Seedream 4.0 [31]",
      "table": "S6.T4",
      "reported_value": "8.0428",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "Seedream 4.0 [31]",
        "8.3764",
        "8.7200",
        "8.0736",
        "7.5960",
        "8.0428",
        "8.3418",
        "8.6600",
        "8.1364",
        "7.1240",
        "8.0474"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "gpt-image-1",
    "bench": "unicbench-en-overall",
    "score": 0.83546,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "GPT-Image-1 [17]; S6.T4 Overall-EN 8.3546 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 8.3546/10. Closed model. Table label GPT-Image-1; the setup prose calls it GPT-4o [17].",
    "evidence": {
      "source_model_label": "GPT-Image-1 [17]",
      "table": "S6.T4",
      "reported_value": "8.3546",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "GPT-Image-1 [17]",
        "9.1551",
        "7.8449",
        "8.6830",
        "8.3392",
        "8.3546",
        "9.2759",
        "7.8906",
        "8.6980",
        "8.2247",
        "8.4506"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "nano-banana-pro",
    "bench": "tecci-human-overall-success",
    "score": 0.223,
    "source": "https://arxiv.org/html/2606.01213v1#S4.T2",
    "source_id": "visual-tecci",
    "by": "lab",
    "date": "2026-05-31",
    "note": "Nano Banana Pro; S4.T2 IRCS+GGIS Overall 22.3% (human, tau=4.5 on IF/IC/VQ); official API, default hyper-parameters. Google evaluating its own model: lab.",
    "evidence": {
      "source_model_label": "Nano Banana Pro",
      "table": "S4.T2",
      "reported_value": "22.3",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "TECCI human study; 1,315 pairs; five raters; Overall = all three criteria >= 4.5.",
      "source_row": [
        "Nano Banana Pro",
        "39.0",
        "34.5",
        "36.0",
        "19.7",
        "47.7",
        "44.0",
        "38.5",
        "23.0",
        "46.0",
        "42.1",
        "38.0",
        "22.3"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.01213v1#S4.SS1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "TECCI authors (Google Research / Google DeepMind)",
    "source_group": "visual-tecci:tecci-human-overall-success"
  },
  {
    "model": "nano-banana-2",
    "bench": "tecci-human-overall-success",
    "score": 0.197,
    "source": "https://arxiv.org/html/2606.01213v1#S4.T2",
    "source_id": "visual-tecci",
    "by": "lab",
    "date": "2026-05-31",
    "note": "Nano Banana 2; S4.T2 IRCS+GGIS Overall 19.7% (human, tau=4.5 on IF/IC/VQ); official API, default hyper-parameters. Google evaluating its own model: lab.",
    "evidence": {
      "source_model_label": "Nano Banana 2",
      "table": "S4.T2",
      "reported_value": "19.7",
      "divisor": 100,
      "measurement_origin": "provider own-model evaluation",
      "protocol": "TECCI human study; 1,315 pairs; five raters; Overall = all three criteria >= 4.5.",
      "source_row": [
        "Nano Banana 2",
        "37.7",
        "30.6",
        "37.7",
        "17.7",
        "47.6",
        "36.5",
        "33.6",
        "20.2",
        "45.6",
        "35.3",
        "34.4",
        "19.7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.01213v1#S4.SS1.p1",
      "evaluator_relationship": "model author evaluating own model"
    },
    "evaluator": "TECCI authors (Google Research / Google DeepMind)",
    "source_group": "visual-tecci:tecci-human-overall-success"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "tecci-human-overall-success",
    "score": 0.057,
    "source": "https://arxiv.org/html/2606.01213v1#S4.T2",
    "source_id": "visual-tecci",
    "by": "independent",
    "date": "2026-05-31",
    "note": "GPT Image 1.5; S4.T2 IRCS+GGIS Overall 5.7% (human, tau=4.5 on IF/IC/VQ); official API, default hyper-parameters. HTML cell rendered with a phantom leading zero ('05.7').",
    "evidence": {
      "source_model_label": "GPT Image 1.5",
      "table": "S4.T2",
      "reported_value": "5.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TECCI human study; 1,315 pairs; five raters; Overall = all three criteria >= 4.5.",
      "source_row": [
        "GPT Image 1.5",
        "29.9",
        "08.3",
        "20.8",
        "04.9",
        "45.0",
        "09.9",
        "21.1",
        "05.9",
        "42.0",
        "09.6",
        "21.0",
        "05.7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.01213v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "TECCI authors (Google Research / Google DeepMind)",
    "source_group": "visual-tecci:tecci-human-overall-success"
  },
  {
    "model": "grok-imagine-image-pro",
    "bench": "tecci-human-overall-success",
    "score": 0.2,
    "source": "https://arxiv.org/html/2606.01213v1#S4.T2",
    "source_id": "visual-tecci",
    "by": "independent",
    "date": "2026-05-31",
    "note": "Grok Imagine Pro; S4.T2 IRCS+GGIS Overall 20.0% (human, tau=4.5 on IF/IC/VQ); official API, default hyper-parameters.",
    "evidence": {
      "source_model_label": "Grok Imagine Pro",
      "table": "S4.T2",
      "reported_value": "20.0",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TECCI human study; 1,315 pairs; five raters; Overall = all three criteria >= 4.5.",
      "source_row": [
        "Grok Imagine Pro",
        "37.4",
        "33.2",
        "33.6",
        "14.7",
        "49.3",
        "40.8",
        "37.3",
        "21.3",
        "46.9",
        "39.2",
        "36.6",
        "20.0"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.01213v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "TECCI authors (Google Research / Google DeepMind)",
    "source_group": "visual-tecci:tecci-human-overall-success"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "tecci-human-overall-success",
    "score": 0.167,
    "source": "https://arxiv.org/html/2606.01213v1#S4.T2",
    "source_id": "visual-tecci",
    "by": "independent",
    "date": "2026-05-31",
    "note": "Seedream 5.0 Lite; S4.T2 IRCS+GGIS Overall 16.7% (human, tau=4.5 on IF/IC/VQ); official API, default hyper-parameters.",
    "evidence": {
      "source_model_label": "Seedream 5.0 Lite",
      "table": "S4.T2",
      "reported_value": "16.7",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "TECCI human study; 1,315 pairs; five raters; Overall = all three criteria >= 4.5.",
      "source_row": [
        "Seedream 5.0 Lite",
        "32.6",
        "29.5",
        "21.1",
        "10.3",
        "43.6",
        "40.6",
        "30.3",
        "18.2",
        "41.4",
        "38.4",
        "28.5",
        "16.7"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.01213v1#S4.SS1.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "TECCI authors (Google Research / Google DeepMind)",
    "source_group": "visual-tecci:tecci-human-overall-success"
  },
  {
    "model": "runway-gen-3-alpha",
    "bench": "skyreels-bench-t2v-human",
    "score": 0.506,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T5",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Runway-Gen3 Alpha; Table 5 Average 2.53/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Runway-Gen3 Alpha [74]",
      "table": "S5.T5",
      "reported_value": "2.53",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Runway-Gen3 Alpha [74]",
        "2.53",
        "2.19",
        "2.57",
        "3.23",
        "2.11"
      ],
      "source_header": [
        "Model Name",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-t2v-human"
  },
  {
    "model": "hunyuanvideo",
    "bench": "skyreels-bench-t2v-human",
    "score": 0.564,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T5",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "HunyuanVideo-13B; Table 5 Average 2.82/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "HunyuanVideo-13B [18]",
      "table": "S5.T5",
      "reported_value": "2.82",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "HunyuanVideo-13B [18]",
        "2.82",
        "2.64",
        "2.81",
        "3.20",
        "2.61"
      ],
      "source_header": [
        "Model Name",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-t2v-human"
  },
  {
    "model": "kling-1-6-std",
    "bench": "skyreels-bench-t2v-human",
    "score": 0.598,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T5",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Kling-1.6 STD Mode; Table 5 Average 2.99/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Kling-1.6 STD Mode [2]",
      "table": "S5.T5",
      "reported_value": "2.99",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Kling-1.6 STD Mode [2]",
        "2.99",
        "2.77",
        "3.05",
        "3.39",
        "2.76"
      ],
      "source_header": [
        "Model Name",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-t2v-human"
  },
  {
    "model": "hailuo-01",
    "bench": "skyreels-bench-t2v-human",
    "score": 0.6,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T5",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Hailuo-01; Table 5 Average 3.0/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings); MiniMax Hailuo-01 T2V (video-01); T2V and I2V endpoints of this closed release share one catalogue id, as for hailuo-2-3. Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Hailuo-01 [3]",
      "table": "S5.T5",
      "reported_value": "3.0",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Hailuo-01 [3]",
        "3.0",
        "2.8",
        "3.08",
        "3.29",
        "2.74"
      ],
      "source_header": [
        "Model Name",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-t2v-human"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "skyreels-bench-t2v-human",
    "score": 0.624,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T5",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Wan2.1-14B; Table 5 Average 3.12/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings); open-weight Wan2.1 T2V-14B (cited Wan report; the same label appears in the authors' open-source VBench comparison), not the wanx2.1 API. Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Wan2.1-14B [5]",
      "table": "S5.T5",
      "reported_value": "3.12",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Wan2.1-14B [5]",
        "3.12",
        "2.91",
        "3.31",
        "3.54",
        "2.71"
      ],
      "source_header": [
        "Model Name",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-t2v-human"
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "vbench-t2v-total-long-prompts",
    "score": 0.803,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T6",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "CogVideoX1.5-5B; Table 6 Total Score 80.3 % (VBench 1.0 long-prompt version, authors' own inference with the baseline's recommended settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "CogVideoX1.5-5B [17]",
      "table": "S5.T6",
      "reported_value": "80.3 %",
      "divisor": 100,
      "measurement_origin": "explicit original VBench run by the paper authors",
      "protocol": "VBench 1.0 T2V long-prompt version; baselines inferred with their recommended settings; published Total Score.",
      "source_row": [
        "CogVideoX1.5-5B [17]",
        "80.3 %",
        "80.9 %",
        "77.9 %"
      ],
      "source_header": [
        "Model",
        "Total Score",
        "Quality Score",
        "Semantic Score"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS2.p2.1",
      "provenance_quote": "we conduct comprehensive evaluations using the public benchmark VBench1.0 ... Our evaluation specifically leverages the benchmark's longer version prompt. For fair comparison with baseline models, we strictly follow their recommended setting for inference.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:vbench-t2v-total-long-prompts"
  },
  {
    "model": "open-sora-2-0",
    "bench": "vbench-t2v-total-long-prompts",
    "score": 0.815,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T6",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "OpenSora-2.0; Table 6 Total Score 81.5 % (VBench 1.0 long-prompt version, authors' own inference with the baseline's recommended settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "OpenSora-2.0 [75]",
      "table": "S5.T6",
      "reported_value": "81.5 %",
      "divisor": 100,
      "measurement_origin": "explicit original VBench run by the paper authors",
      "protocol": "VBench 1.0 T2V long-prompt version; baselines inferred with their recommended settings; published Total Score.",
      "source_row": [
        "OpenSora-2.0 [75]",
        "81.5 %",
        "82.1 %",
        "78.2 %"
      ],
      "source_header": [
        "Model",
        "Total Score",
        "Quality Score",
        "Semantic Score"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS2.p2.1",
      "provenance_quote": "we conduct comprehensive evaluations using the public benchmark VBench1.0 ... Our evaluation specifically leverages the benchmark's longer version prompt. For fair comparison with baseline models, we strictly follow their recommended setting for inference.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:vbench-t2v-total-long-prompts"
  },
  {
    "model": "hunyuanvideo",
    "bench": "vbench-t2v-total-long-prompts",
    "score": 0.827,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T6",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "HunyuanVideo-13B; Table 6 Total Score 82.7 % (VBench 1.0 long-prompt version, authors' own inference with the baseline's recommended settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "HunyuanVideo-13B [18]",
      "table": "S5.T6",
      "reported_value": "82.7 %",
      "divisor": 100,
      "measurement_origin": "explicit original VBench run by the paper authors",
      "protocol": "VBench 1.0 T2V long-prompt version; baselines inferred with their recommended settings; published Total Score.",
      "source_row": [
        "HunyuanVideo-13B [18]",
        "82.7 %",
        "84.4 %",
        "76.2 %"
      ],
      "source_header": [
        "Model",
        "Total Score",
        "Quality Score",
        "Semantic Score"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS2.p2.1",
      "provenance_quote": "we conduct comprehensive evaluations using the public benchmark VBench1.0 ... Our evaluation specifically leverages the benchmark's longer version prompt. For fair comparison with baseline models, we strictly follow their recommended setting for inference.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:vbench-t2v-total-long-prompts"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-total-long-prompts",
    "score": 0.837,
    "source": "https://arxiv.org/html/2504.13074v3#S5.T6",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Wan2.1-14B; Table 6 Total Score 83.7 % (VBench 1.0 long-prompt version, authors' own inference with the baseline's recommended settings); open-weight Wan2.1 T2V-14B; Section 5.2 compares 'other leading open-source video generation models'. Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Wan2.1-14B [5]",
      "table": "S5.T6",
      "reported_value": "83.7 %",
      "divisor": 100,
      "measurement_origin": "explicit original VBench run by the paper authors",
      "protocol": "VBench 1.0 T2V long-prompt version; baselines inferred with their recommended settings; published Total Score.",
      "source_row": [
        "Wan2.1-14B [5]",
        "83.7 %",
        "84.2 %",
        "81.4 %"
      ],
      "source_header": [
        "Model",
        "Total Score",
        "Quality Score",
        "Semantic Score"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS2.p2.1",
      "provenance_quote": "we conduct comprehensive evaluations using the public benchmark VBench1.0 ... Our evaluation specifically leverages the benchmark's longer version prompt. For fair comparison with baseline models, we strictly follow their recommended setting for inference.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:vbench-t2v-total-long-prompts"
  },
  {
    "model": "hunyuanvideo-i2v",
    "bench": "skyreels-bench-i2v-human",
    "score": 0.568,
    "source": "https://arxiv.org/html/2504.13074v3#S6.T7",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "HunyuanVideo-13B; Table 7 Average 2.84/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings); row labelled HunyuanVideo-13B in the I2V table; the 13B HunyuanVideo I2V checkpoint is HunyuanVideo-I2V (the T2V release has no image conditioning). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "HunyuanVideo-13B [18]",
      "table": "S6.T7",
      "reported_value": "2.84",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "HunyuanVideo-13B [18]",
        "2.84",
        "2.97",
        "2.95",
        "2.87",
        "2.56"
      ],
      "source_header": [
        "Model",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-i2v-human"
  },
  {
    "model": "hailuo-01",
    "bench": "skyreels-bench-i2v-human",
    "score": 0.61,
    "source": "https://arxiv.org/html/2504.13074v3#S6.T7",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Hailuo-01; Table 7 Average 3.05/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings); MiniMax Hailuo-01 I2V (I2V-01); same closed release id as the T2V row. Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Hailuo-01 [3]",
      "table": "S6.T7",
      "reported_value": "3.05",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Hailuo-01 [3]",
        "3.05",
        "3.31",
        "2.58",
        "3.55",
        "2.74"
      ],
      "source_header": [
        "Model",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-i2v-human"
  },
  {
    "model": "kling-1-6-pro",
    "bench": "skyreels-bench-i2v-human",
    "score": 0.68,
    "source": "https://arxiv.org/html/2504.13074v3#S6.T7",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Kling-1.6 Pro Mode; Table 7 Average 3.4/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Kling-1.6 Pro Mode [2]",
      "table": "S6.T7",
      "reported_value": "3.4",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Kling-1.6 Pro Mode [2]",
        "3.4",
        "3.56",
        "3.03",
        "3.58",
        "3.41"
      ],
      "source_header": [
        "Model",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-i2v-human"
  },
  {
    "model": "runway-gen-4",
    "bench": "skyreels-bench-i2v-human",
    "score": 0.678,
    "source": "https://arxiv.org/html/2504.13074v3#S6.T7",
    "source_id": "visual-skyreels-v2",
    "by": "independent",
    "date": "2025-04-21",
    "note": "Runway-Gen4; Table 7 Average 3.39/5, SkyReels-Bench human rating (20 professional evaluators, 1,020 prompts, default settings). Reported release/mode label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Runway-Gen4 [12]",
      "table": "S6.T7",
      "reported_value": "3.39",
      "divisor": 5,
      "measurement_origin": "explicit original human evaluation by the paper authors",
      "protocol": "SkyReels-Bench; 20 professional evaluators; 1–5 per dimension; published Average of four dimensions; default settings, consistent resolutions, no post-filtering.",
      "source_row": [
        "Runway-Gen4 [12]",
        "3.39",
        "3.75",
        "3.2",
        "3.4",
        "3.37"
      ],
      "source_header": [
        "Model",
        "Average",
        "Instruction Adherence",
        "Consistency",
        "Visual Quality",
        "Motion Quality"
      ],
      "date_basis": "Report version date (v3, 2025-04-21), not an asserted generation date",
      "model_identity_scope": "Source-reported release/mode label; no checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2504.13074v3#S5.SS1.SSS0.Px4.p4.1",
      "provenance_quote": "For evaluation, a panel of 20 professional evaluators assess each dimension using a 1-5 scale ... To ensure fairness, all models are evaluated under default settings with consistent resolutions, and no post-generation filtering is applied.",
      "evaluator_relationship": "competing model developer (Skywork AI) evaluating other providers' models"
    },
    "evaluator": "SkyReels Team / Skywork AI",
    "source_group": "visual-skyreels-v2:skyreels-bench-i2v-human"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "videophy2-human-joint",
    "score": 0.326,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Wan2.1-T2V-14B; VideoPhy-2 human leaderboard 'All' 32.6% (SA>=4 and PC>=4); linked checkpoint Wan-AI/Wan2.1-T2V-14B. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "Wan2.1-T2V-14B",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "32.6",
      "divisor": 100,
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "protocol": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "source_row": [
        "1",
        "[Wan2.1-T2V-14B](https://huggingface.co/Wan-AI/Wan2.1-T2V-14B)🥇",
        "Open",
        "32.6",
        "21.9"
      ],
      "source_header": [
        "#",
        "Model",
        "Source",
        "All",
        "Hard"
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
      "provenance_quote": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy2-human-joint"
  },
  {
    "model": "cogvideox-5b",
    "bench": "videophy2-human-joint",
    "score": 0.25,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "CogVideoX-5B; VideoPhy-2 human leaderboard 'All' 25.0% (SA>=4 and PC>=4); linked checkpoint THUDM/CogVideoX-5b. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "CogVideoX-5B",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "25.0",
      "divisor": 100,
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "protocol": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "source_row": [
        "2",
        "[CogVideoX-5B](https://huggingface.co/THUDM/CogVideoX-5b)🥈",
        "Open",
        "25.0",
        "0"
      ],
      "source_header": [
        "#",
        "Model",
        "Source",
        "All",
        "Hard"
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
      "provenance_quote": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy2-human-joint"
  },
  {
    "model": "cosmos-1-0-diffusion-7b-text2world",
    "bench": "videophy2-human-joint",
    "score": 0.241,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Cosmos-Diffusion-7B; VideoPhy-2 human leaderboard 'All' 24.1% (SA>=4 and PC>=4); linked checkpoint nvidia/Cosmos-1.0-Diffusion-7B-Text2World. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "Cosmos-Diffusion-7B",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "24.1",
      "divisor": 100,
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "protocol": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "source_row": [
        "3",
        "[Cosmos-Diffusion-7B](https://huggingface.co/nvidia/Cosmos-1.0-Diffusion-7B-Text2World)🥉",
        "Open",
        "24.1",
        "10.9"
      ],
      "source_header": [
        "#",
        "Model",
        "Source",
        "All",
        "Hard"
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
      "provenance_quote": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy2-human-joint"
  },
  {
    "model": "luma-ray-2",
    "bench": "videophy2-human-joint",
    "score": 0.203,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Luma Ray2; VideoPhy-2 human leaderboard 'All' 20.3% (SA>=4 and PC>=4); Luma Ray2 (closed; generation settings not stated). Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "Luma Ray2",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "20.3",
      "divisor": 100,
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "protocol": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "source_row": [
        "5",
        "[Luma Ray2](https://lumalabs.ai/ray)",
        "Closed",
        "20.3",
        "8.3"
      ],
      "source_header": [
        "#",
        "Model",
        "Source",
        "All",
        "Hard"
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
      "provenance_quote": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy2-human-joint"
  },
  {
    "model": "hunyuanvideo",
    "bench": "videophy2-human-joint",
    "score": 0.172,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "HunyuanVideo-13B; VideoPhy-2 human leaderboard 'All' 17.2% (SA>=4 and PC>=4); linked checkpoint tencent/HunyuanVideo. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "HunyuanVideo-13B",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "17.2",
      "divisor": 100,
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "protocol": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "source_row": [
        "6",
        "[HunyuanVideo-13B](https://huggingface.co/tencent/HunyuanVideo)",
        "Open",
        "17.2",
        "6.2"
      ],
      "source_header": [
        "#",
        "Model",
        "Source",
        "All",
        "Hard"
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
      "provenance_quote": "We evaluate 7 closed and open text-to-video generative models on VideoPhy-2 dataset with human annotation. We report the joint performance. It is computed by aggregating the percentage of testing prompts for which the T2V models generate videos that adhere to the conditioning caption (SA>=4) and exhibit high physical commonsense (PC>=4).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy2-human-joint"
  },
  {
    "model": "cogvideox-5b",
    "bench": "videophy-autoeval-avg",
    "score": 0.49,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "CogVideoX-5B; VideoPhy-1 automatic leaderboard 'Avg.' 49; linked repo THUDM/CogVideo, 5B. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "CogVideoX-5B",
      "table": "README.md leaderboard",
      "reported_value": "49",
      "divisor": 100,
      "measurement_origin": "explicit original auto-evaluator run by the benchmark authors",
      "protocol": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "source_row": [
        "1",
        "[CogVideoX-5B](https://github.com/THUDM/CogVideo)🥇",
        "41",
        "57",
        "49"
      ],
      "source_header": [
        "#",
        "Model",
        "PC=1",
        "SA=1",
        "Avg."
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
      "provenance_quote": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg"
  },
  {
    "model": "mochi-1",
    "bench": "videophy-autoeval-avg",
    "score": 0.41,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Mochi; VideoPhy-1 automatic leaderboard 'Avg.' 41; linked repo genmoai/mochi (Mochi 1). Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "Mochi",
      "table": "README.md leaderboard",
      "reported_value": "41",
      "divisor": 100,
      "measurement_origin": "explicit original auto-evaluator run by the benchmark authors",
      "protocol": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "source_row": [
        "5",
        "[Mochi](https://github.com/genmoai/mochi)",
        "31",
        "50",
        "41"
      ],
      "source_header": [
        "#",
        "Model",
        "PC=1",
        "SA=1",
        "Avg."
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
      "provenance_quote": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg"
  },
  {
    "model": "hunyuanvideo",
    "bench": "videophy-autoeval-avg",
    "score": 0.35,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "by": "independent",
    "date": "2026-01-30",
    "note": "HunyuanVideo; VideoPhy-1 automatic leaderboard 'Avg.' 35; linked repo Tencent/HunyuanVideo. Repository README at commit 01d366c.",
    "evidence": {
      "source_model_label": "HunyuanVideo",
      "table": "README.md leaderboard",
      "reported_value": "35",
      "divisor": 100,
      "measurement_origin": "explicit original auto-evaluator run by the benchmark authors",
      "protocol": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "source_row": [
        "9",
        "[HunyuanVideo](https://github.com/Tencent/HunyuanVideo)",
        "28",
        "42",
        "35"
      ],
      "source_header": [
        "#",
        "Model",
        "PC=1",
        "SA=1",
        "Avg."
      ],
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "model_identity_scope": "Checkpoint/product linked from the leaderboard row; no snapshot hash claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
      "provenance_quote": "We evaluate 12 closed and open text-to-video generative models on VideoPhy dataset with our own auto evaluator (see our paper for more details).",
      "evaluator_relationship": "benchmark authors evaluating other providers' models"
    },
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg"
  },
  {
    "model": "seedance-1-0-lite",
    "bench": "v-reasonbench-pass5-avg",
    "score": 0.1068,
    "source": "https://arxiv.org/html/2511.16668v1#S3.T2",
    "source_id": "visual-v-reasonbench",
    "by": "independent",
    "date": "2025-11-20",
    "note": "Seedance-1.0-Lite; Table 2 Average pass@5 10.68% over four reasoning dimensions; five videos per prompt, default parameters, ~5 s, 720p/768p when supported. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Seedance-1.0-Lite [11]",
      "table": "S3.T2",
      "reported_value": "10.68",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Standardized protocol: five videos per prompt, default parameter settings, ~5 s, 720p or 768p when supported, 16:9; identical prompts; pass@5 per dimension; Average of four dimensions.",
      "source_row": [
        "Seedance-1.0-Lite [11]",
        "0.57",
        "8.82",
        "0.00",
        "33.33",
        "10.68"
      ],
      "source_header": [
        "Model",
        "Structured Problem-Solving",
        "Spatial Cognition",
        "Pattern-based Inference",
        "Physical Dynamics",
        "Average"
      ],
      "date_basis": "Report version date (v1, 2025-11-20), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.16668v1#S4.SS1.p2.1",
      "provenance_quote": "We evaluate six prevalent generative video models ... Sora-2, Veo-3.1, Hailuo-02, Vidu-Q2, KlingAI-2.5-Turbo-Pro, and Seedance-1.0-Lite. (S4.1) We evaluate each model using a standardized protocol that generates five videos per prompt. All models are run with their default parameter settings ...",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "V-ReasonBench authors (NUS / HKUST(GZ) / HKU / USYD / CUHK / LIGHTSPEED)",
    "source_group": "visual-v-reasonbench:v-reasonbench-pass5-avg"
  },
  {
    "model": "kling-2-5-turbo-pro",
    "bench": "v-reasonbench-pass5-avg",
    "score": 0.1134,
    "source": "https://arxiv.org/html/2511.16668v1#S3.T2",
    "source_id": "visual-v-reasonbench",
    "by": "independent",
    "date": "2025-11-20",
    "note": "Kling-2.5-Turbo-Pro; Table 2 Average pass@5 11.34% over four reasoning dimensions; five videos per prompt, default parameters, ~5 s, 720p/768p when supported; KlingAI-2.5-Turbo-Pro in Section 4.1. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Kling-2.5-Turbo-Pro [19]",
      "table": "S3.T2",
      "reported_value": "11.34",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Standardized protocol: five videos per prompt, default parameter settings, ~5 s, 720p or 768p when supported, 16:9; identical prompts; pass@5 per dimension; Average of four dimensions.",
      "source_row": [
        "Kling-2.5-Turbo-Pro [19]",
        "5.14",
        "13.24",
        "3.64",
        "23.33",
        "11.34"
      ],
      "source_header": [
        "Model",
        "Structured Problem-Solving",
        "Spatial Cognition",
        "Pattern-based Inference",
        "Physical Dynamics",
        "Average"
      ],
      "date_basis": "Report version date (v1, 2025-11-20), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.16668v1#S4.SS1.p2.1",
      "provenance_quote": "We evaluate six prevalent generative video models ... Sora-2, Veo-3.1, Hailuo-02, Vidu-Q2, KlingAI-2.5-Turbo-Pro, and Seedance-1.0-Lite. (S4.1) We evaluate each model using a standardized protocol that generates five videos per prompt. All models are run with their default parameter settings ...",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "V-ReasonBench authors (NUS / HKUST(GZ) / HKU / USYD / CUHK / LIGHTSPEED)",
    "source_group": "visual-v-reasonbench:v-reasonbench-pass5-avg"
  },
  {
    "model": "veo-3-1",
    "bench": "v-reasonbench-pass5-avg",
    "score": 0.2425,
    "source": "https://arxiv.org/html/2511.16668v1#S3.T2",
    "source_id": "visual-v-reasonbench",
    "by": "independent",
    "date": "2025-11-20",
    "note": "Veo-3.1; Table 2 Average pass@5 24.25% over four reasoning dimensions; five videos per prompt, default parameters, ~5 s, 720p/768p when supported. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Veo-3.1 [14]",
      "table": "S3.T2",
      "reported_value": "24.25",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Standardized protocol: five videos per prompt, default parameter settings, ~5 s, 720p or 768p when supported, 16:9; identical prompts; pass@5 per dimension; Average of four dimensions.",
      "source_row": [
        "Veo-3.1 [14]",
        "26.29",
        "26.47",
        "10.91",
        "33.33",
        "24.25"
      ],
      "source_header": [
        "Model",
        "Structured Problem-Solving",
        "Spatial Cognition",
        "Pattern-based Inference",
        "Physical Dynamics",
        "Average"
      ],
      "date_basis": "Report version date (v1, 2025-11-20), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.16668v1#S4.SS1.p2.1",
      "provenance_quote": "We evaluate six prevalent generative video models ... Sora-2, Veo-3.1, Hailuo-02, Vidu-Q2, KlingAI-2.5-Turbo-Pro, and Seedance-1.0-Lite. (S4.1) We evaluate each model using a standardized protocol that generates five videos per prompt. All models are run with their default parameter settings ...",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "V-ReasonBench authors (NUS / HKUST(GZ) / HKU / USYD / CUHK / LIGHTSPEED)",
    "source_group": "visual-v-reasonbench:v-reasonbench-pass5-avg"
  },
  {
    "model": "hailuo-02",
    "bench": "v-reasonbench-pass5-avg",
    "score": 0.3752,
    "source": "https://arxiv.org/html/2511.16668v1#S3.T2",
    "source_id": "visual-v-reasonbench",
    "by": "independent",
    "date": "2025-11-20",
    "note": "Hailuo-02; Table 2 Average pass@5 37.52% over four reasoning dimensions; five videos per prompt, default parameters, ~5 s, 720p/768p when supported; 768p used when supported. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Hailuo-02 [26]",
      "table": "S3.T2",
      "reported_value": "37.52",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Standardized protocol: five videos per prompt, default parameter settings, ~5 s, 720p or 768p when supported, 16:9; identical prompts; pass@5 per dimension; Average of four dimensions.",
      "source_row": [
        "Hailuo-02 [26]",
        "46.86",
        "33.82",
        "32.73",
        "36.67",
        "37.52"
      ],
      "source_header": [
        "Model",
        "Structured Problem-Solving",
        "Spatial Cognition",
        "Pattern-based Inference",
        "Physical Dynamics",
        "Average"
      ],
      "date_basis": "Report version date (v1, 2025-11-20), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.16668v1#S4.SS1.p2.1",
      "provenance_quote": "We evaluate six prevalent generative video models ... Sora-2, Veo-3.1, Hailuo-02, Vidu-Q2, KlingAI-2.5-Turbo-Pro, and Seedance-1.0-Lite. (S4.1) We evaluate each model using a standardized protocol that generates five videos per prompt. All models are run with their default parameter settings ...",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "V-ReasonBench authors (NUS / HKUST(GZ) / HKU / USYD / CUHK / LIGHTSPEED)",
    "source_group": "visual-v-reasonbench:v-reasonbench-pass5-avg"
  },
  {
    "model": "sora-2",
    "bench": "v-reasonbench-pass5-avg",
    "score": 0.4386,
    "source": "https://arxiv.org/html/2511.16668v1#S3.T2",
    "source_id": "visual-v-reasonbench",
    "by": "independent",
    "date": "2025-11-20",
    "note": "Sora-2; Table 2 Average pass@5 43.86% over four reasoning dimensions; five videos per prompt, default parameters, ~5 s, 720p/768p when supported. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Sora-2 [27]",
      "table": "S3.T2",
      "reported_value": "43.86",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Standardized protocol: five videos per prompt, default parameter settings, ~5 s, 720p or 768p when supported, 16:9; identical prompts; pass@5 per dimension; Average of four dimensions.",
      "source_row": [
        "Sora-2 [27]",
        "72.00",
        "36.76",
        "40.00",
        "26.67",
        "43.86"
      ],
      "source_header": [
        "Model",
        "Structured Problem-Solving",
        "Spatial Cognition",
        "Pattern-based Inference",
        "Physical Dynamics",
        "Average"
      ],
      "date_basis": "Report version date (v1, 2025-11-20), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2511.16668v1#S4.SS1.p2.1",
      "provenance_quote": "We evaluate six prevalent generative video models ... Sora-2, Veo-3.1, Hailuo-02, Vidu-Q2, KlingAI-2.5-Turbo-Pro, and Seedance-1.0-Lite. (S4.1) We evaluate each model using a standardized protocol that generates five videos per prompt. All models are run with their default parameter settings ...",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "V-ReasonBench authors (NUS / HKUST(GZ) / HKU / USYD / CUHK / LIGHTSPEED)",
    "source_group": "visual-v-reasonbench:v-reasonbench-pass5-avg"
  },
  {
    "model": "wan-2-5-preview",
    "bench": "vwg-bench-overall",
    "score": 0.55,
    "source": "https://arxiv.org/html/2609.11242v1#S4.T2",
    "source_id": "visual-vwg-bench",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Wan2.5; Table 2 Overall 2.75/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge; commercial Wan2.5 with built-in prompt extension disabled; Alibaba's Wan 2.5 API is published only as the preview release. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Wan2.5",
      "table": "S4.T2",
      "reported_value": "2.75",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Wan2.5",
        "2.70",
        "2.86",
        "3.14",
        "2.48",
        "2.56",
        "2.75"
      ],
      "source_header": [
        "Model",
        "Video Quality",
        "Prog. Consist.",
        "Rule Follow.",
        "Prog. Goal",
        "Last Goal",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "benchmark authors (incl. Kling Team, a competing provider) evaluating other providers' models"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "wan-2-6-flash",
    "bench": "vwg-bench-overall",
    "score": 0.56,
    "source": "https://arxiv.org/html/2609.11242v1#S4.T2",
    "source_id": "visual-vwg-bench",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Wan2.6-Flash; Table 2 Overall 2.80/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge; commercial Wan2.6-Flash, prompt extension disabled. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Wan2.6-Flash",
      "table": "S4.T2",
      "reported_value": "2.80",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Wan2.6-Flash",
        "2.73",
        "2.92",
        "3.23",
        "2.50",
        "2.62",
        "2.80"
      ],
      "source_header": [
        "Model",
        "Video Quality",
        "Prog. Consist.",
        "Rule Follow.",
        "Prog. Goal",
        "Last Goal",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "benchmark authors (incl. Kling Team, a competing provider) evaluating other providers' models"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "kling-2-5-turbo-pro",
    "bench": "vwg-bench-overall",
    "score": 0.594,
    "source": "https://arxiv.org/html/2609.11242v1#S4.T2",
    "source_id": "visual-vwg-bench",
    "by": "lab",
    "date": "2026-09-10",
    "note": "Kling-2.5-Turbo-Pro; Table 2 Overall 2.97/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge; evaluated by authors including the Kling Team (Kuaishou). Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Kling-2.5-Turbo-Pro",
      "table": "S4.T2",
      "reported_value": "2.97",
      "divisor": 5,
      "measurement_origin": "evaluation by authors affiliated with the model maker",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Kling-2.5-Turbo-Pro",
        "2.95",
        "3.12",
        "3.38",
        "2.62",
        "2.78",
        "2.97"
      ],
      "source_header": [
        "Model",
        "Video Quality",
        "Prog. Consist.",
        "Rule Follow.",
        "Prog. Goal",
        "Last Goal",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "model maker's team among the evaluators"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "sora-2",
    "bench": "vwg-bench-overall",
    "score": 0.688,
    "source": "https://arxiv.org/html/2609.11242v1#S4.T2",
    "source_id": "visual-vwg-bench",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Sora2; Table 2 Overall 3.44/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Sora2",
      "table": "S4.T2",
      "reported_value": "3.44",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Sora2",
        "3.45",
        "3.68",
        "3.79",
        "3.32",
        "2.98",
        "3.44"
      ],
      "source_header": [
        "Model",
        "Video Quality",
        "Prog. Consist.",
        "Rule Follow.",
        "Prog. Goal",
        "Last Goal",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "benchmark authors (incl. Kling Team, a competing provider) evaluating other providers' models"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "veo-3-1",
    "bench": "vwg-bench-overall",
    "score": 0.71,
    "source": "https://arxiv.org/html/2609.11242v1#S4.T2",
    "source_id": "visual-vwg-bench",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Veo3.1; Table 2 Overall 3.55/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge. Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Veo3.1",
      "table": "S4.T2",
      "reported_value": "3.55",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Veo3.1",
        "3.54",
        "3.78",
        "3.95",
        "3.45",
        "3.05",
        "3.55"
      ],
      "source_header": [
        "Model",
        "Video Quality",
        "Prog. Consist.",
        "Rule Follow.",
        "Prog. Goal",
        "Last Goal",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "benchmark authors (incl. Kling Team, a competing provider) evaluating other providers' models"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "seedance-2-0",
    "bench": "vwg-bench-overall",
    "score": 0.704,
    "source": "https://arxiv.org/html/2609.11242v1#as1_Pt0.A1.T3",
    "source_id": "visual-vwg-bench",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Seedance2.0; Appendix Table 3 Overall 3.52/5; three generations per image-prompt pair, no prompt extension, Gemini-3.0-Pro judge; Appendix Table 3 under the same protocol (its Veo3.1 row is identical to Table 2). Reported release label retained; no API snapshot claimed.",
    "evidence": {
      "source_model_label": "Seedance2.0",
      "table": "as1_Pt0.A1.T3",
      "reported_value": "3.52",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "VWG-Bench I2V; three generations per (image, prompt) pair; averaged 1–5 judge scores; no prompt extension.",
      "source_row": [
        "Seedance2.0",
        "3.67",
        "3.52",
        "3.87",
        "3.38",
        "3.17",
        "3.52"
      ],
      "source_header": [
        "Model",
        "VQ",
        "PC",
        "RF",
        "PG",
        "LG",
        "Overall"
      ],
      "date_basis": "Report version date (v1, 2026-09-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2609.11242v1#S4.SS1.p1.1",
      "provenance_quote": "We evaluate the general reasoning capability of six video generation models on VWG-Bench, including one open-weight model (Wan2.2) and five commercial systems (Wan2.5, Wan2.6-Flash, Kling-2.5-Turbo-Pro, Sora2, and Veo3.1). For each pair, we sample three generations and report the averaged score for each evaluation dimension.",
      "evaluator_relationship": "benchmark authors (incl. Kling Team, a competing provider) evaluating other providers' models"
    },
    "evaluator": "VWG-Bench authors (NUS / Nanjing University / Kling Team, Kuaishou)",
    "source_group": "visual-vwg-bench:vwg-bench-overall"
  },
  {
    "model": "veo-3-1-fast",
    "bench": "worldjen-human-bt",
    "score": 1614,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "Veo 3.1 Fast; Table 1 human BT 1614 (95% CI 1595-1635); 2,696 votes, 7 annotators, 50 prompts; fal-ai/veo3.1/fast, ≈8 s; generated March 2026.",
    "evidence": {
      "source_model_label": "Veo 3.1 Fast",
      "table": "S5.T1",
      "reported_value": "1614 [1595, 1635]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "Veo 3.1 Fast",
        "1",
        "1614 [1595, 1635]",
        "+0.66+0.66",
        "Top"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "kling-2-6-pro",
    "bench": "worldjen-human-bt",
    "score": 1572,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "Kling v2.6 Pro; Table 1 human BT 1572 (95% CI 1552-1592); 2,696 votes, 7 annotators, 50 prompts; fal-ai/kling-video/v2.6/pro/text-to-video; generated March 2026.",
    "evidence": {
      "source_model_label": "Kling v2.6 Pro",
      "table": "S5.T1",
      "reported_value": "1572 [1552, 1592]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "Kling v2.6 Pro",
        "2",
        "1572 [1552, 1592]",
        "+0.41+0.41",
        "Top"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "worldjen-human-bt",
    "score": 1518,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "Wan v2.2 A14B; Table 1 human BT 1518 (95% CI 1500-1538); 2,696 votes, 7 annotators, 50 prompts; fal-ai/wan/v2.2-a14b/text-to-video; generated March 2026.",
    "evidence": {
      "source_model_label": "Wan v2.2 A14B",
      "table": "S5.T1",
      "reported_value": "1518 [1500, 1538]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "Wan v2.2 A14B",
        "3",
        "1518 [1500, 1538]",
        "+0.10+0.10",
        "Mid"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "ltx-2",
    "bench": "worldjen-human-bt",
    "score": 1479,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "LTX-2; Table 1 human BT 1479 (95% CI 1460-1500); 2,696 votes, 7 annotators, 50 prompts; fal-ai/ltx-2/text-to-video; generated March 2026.",
    "evidence": {
      "source_model_label": "LTX-2",
      "table": "S5.T1",
      "reported_value": "1479 [1460, 1500]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "LTX-2",
        "4",
        "1479 [1460, 1500]",
        "−0.12-0.12",
        "Mid"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "worldjen-human-bt",
    "score": 1462,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "Hunyuan v1.5; Table 1 human BT 1462 (95% CI 1443-1482); 2,696 votes, 7 annotators, 50 prompts; fal-ai/hunyuan-video-v1.5/text-to-video (prose in Section 5 prints 1461; Table 1 value used); generated March 2026.",
    "evidence": {
      "source_model_label": "Hunyuan v1.5",
      "table": "S5.T1",
      "reported_value": "1462 [1443, 1482]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "Hunyuan v1.5",
        "5",
        "1462 [1443, 1482]",
        "−0.22-0.22",
        "Mid"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "wan-2-1-1-3b",
    "bench": "worldjen-human-bt",
    "score": 1355,
    "source": "https://arxiv.org/html/2605.03475v2#S5.T1",
    "source_id": "visual-worldjen",
    "by": "independent",
    "date": "2026-05-06",
    "note": "Wan 2.1 1.3B; Table 1 human BT 1355 (95% CI 1333-1376); 2,696 votes, 7 annotators, 50 prompts; run locally on 8×H200 (open Wan2.1 T2V 1.3B); generated March 2026.",
    "evidence": {
      "source_model_label": "Wan 2.1 1.3B",
      "table": "S5.T1",
      "reported_value": "1355 [1333, 1376]",
      "divisor": 1,
      "measurement_origin": "explicit original generation and human preference study by the benchmark authors",
      "protocol": "50 prompts × 6 models, one video each, default settings; pairwise human votes (7 annotators, full pair coverage) fitted with Bradley-Terry MLE; 95% bootstrap CI.",
      "source_row": [
        "Wan 2.1 1.3B",
        "6",
        "1355 [1333, 1376]",
        "−0.83-0.83",
        "Bottom"
      ],
      "source_header": [
        "Model",
        "Rank",
        "Human BT [95% CI]",
        "BT Strength",
        "Tier"
      ],
      "date_basis": "Report version date (v2, 2026-05-06); videos generated March 2026 per Section 4",
      "model_identity_scope": "Exact fal.ai endpoint or local checkpoint named in Section 4; fal endpoints may update silently (authors release the videos).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.03475v2#S4.SS0.SSS0.Px2.p1.1",
      "provenance_quote": "Six SOTA video models are evaluated: Veo 3.1 Fast, Kling v2.6 Pro, LTX-2, Wan v2.2 A14B, Hunyuan v1.5, and Wan 2.1 1.3B (local open-source baseline). Commercial models are accessed via the fal.ai inference API ... One video is generated per (model, prompt) pair, yielding 300 videos in total.",
      "evaluator_relationship": "independent benchmark authors (moonmath.ai) evaluating other providers' models"
    },
    "evaluator": "WorldJen authors (moonmath.ai)",
    "source_group": "visual-worldjen:worldjen-human-bt"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "t2vphysbench-human-avg",
    "score": 0.42,
    "source": "https://arxiv.org/html/2505.00337v1#S4.T2",
    "source_id": "visual-t2vphysbench",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Wan 2.1; Table 2 Avg. Score 0.42; three human annotators per video, four-level physical-law adherence mapped to [0,1]; Table 1: Wan 2.1, 14B, open (Alibaba).",
    "evidence": {
      "source_model_label": "Wan 2.1",
      "table": "S4.T2",
      "reported_value": "0.42",
      "divisor": 1,
      "measurement_origin": "explicit original human evaluation by the benchmark authors",
      "protocol": "Fully manual evaluation: three annotators review every generated video; four quality levels mapped to [0,1]; averaged across prompts and annotators.",
      "source_row": [
        "Wan 2.1",
        "0.56",
        "0.29",
        "0.42",
        "0.42"
      ],
      "source_header": [
        "Model",
        "Newton Principles",
        "Conservation Principles",
        "Phenomenon Principles",
        "Avg. Score"
      ],
      "model_info_row": [
        "Wan 2.1 [3]",
        "2025",
        "14B",
        "Alibaba",
        "Yes"
      ],
      "date_basis": "Report version date (v1, 2025-05-01), not an asserted generation date",
      "model_identity_scope": "Release identified by Table 1 (year, parameters, organisation, openness); no API snapshot claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2505.00337v1#S3.SS3.p1.1",
      "provenance_quote": "we adopt a fully manual evaluation protocol, following VideoPhy. Three annotators (undergraduate or graduate students) independently review every generated video ... We compare all the models listed in Table 1 and present the overall result in Table 2.",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "T2VPhysBench authors",
    "source_group": "visual-t2vphysbench:t2vphysbench-human-avg"
  },
  {
    "model": "sora",
    "bench": "t2vphysbench-human-avg",
    "score": 0.28,
    "source": "https://arxiv.org/html/2505.00337v1#S4.T2",
    "source_id": "visual-t2vphysbench",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Sora; Table 2 Avg. Score 0.28; three human annotators per video, four-level physical-law adherence mapped to [0,1]; Table 1: Sora (OpenAI, 2024), the original Sora release.",
    "evidence": {
      "source_model_label": "Sora",
      "table": "S4.T2",
      "reported_value": "0.28",
      "divisor": 1,
      "measurement_origin": "explicit original human evaluation by the benchmark authors",
      "protocol": "Fully manual evaluation: three annotators review every generated video; four quality levels mapped to [0,1]; averaged across prompts and annotators.",
      "source_row": [
        "Sora",
        "0.31",
        "0.15",
        "0.38",
        "0.28"
      ],
      "source_header": [
        "Model",
        "Newton Principles",
        "Conservation Principles",
        "Phenomenon Principles",
        "Avg. Score"
      ],
      "model_info_row": [
        "Sora [42]",
        "2024",
        "N/A",
        "OpenAI",
        "No"
      ],
      "date_basis": "Report version date (v1, 2025-05-01), not an asserted generation date",
      "model_identity_scope": "Release identified by Table 1 (year, parameters, organisation, openness); no API snapshot claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2505.00337v1#S3.SS3.p1.1",
      "provenance_quote": "we adopt a fully manual evaluation protocol, following VideoPhy. Three annotators (undergraduate or graduate students) independently review every generated video ... We compare all the models listed in Table 1 and present the overall result in Table 2.",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "T2VPhysBench authors",
    "source_group": "visual-t2vphysbench:t2vphysbench-human-avg"
  },
  {
    "model": "mochi-1",
    "bench": "t2vphysbench-human-avg",
    "score": 0.34,
    "source": "https://arxiv.org/html/2505.00337v1#S4.T2",
    "source_id": "visual-t2vphysbench",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Mochi-1; Table 2 Avg. Score 0.34; three human annotators per video, four-level physical-law adherence mapped to [0,1]; Table 1: Mochi-1, 10B, Genmo.",
    "evidence": {
      "source_model_label": "Mochi-1",
      "table": "S4.T2",
      "reported_value": "0.34",
      "divisor": 1,
      "measurement_origin": "explicit original human evaluation by the benchmark authors",
      "protocol": "Fully manual evaluation: three annotators review every generated video; four quality levels mapped to [0,1]; averaged across prompts and annotators.",
      "source_row": [
        "Mochi-1",
        "0.40",
        "0.23",
        "0.40",
        "0.34"
      ],
      "source_header": [
        "Model",
        "Newton Principles",
        "Conservation Principles",
        "Phenomenon Principles",
        "Avg. Score"
      ],
      "model_info_row": [
        "Mochi-1 [16]",
        "2024",
        "10B",
        "Genmo",
        "Yes"
      ],
      "date_basis": "Report version date (v1, 2025-05-01), not an asserted generation date",
      "model_identity_scope": "Release identified by Table 1 (year, parameters, organisation, openness); no API snapshot claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2505.00337v1#S3.SS3.p1.1",
      "provenance_quote": "we adopt a fully manual evaluation protocol, following VideoPhy. Three annotators (undergraduate or graduate students) independently review every generated video ... We compare all the models listed in Table 1 and present the overall result in Table 2.",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "T2VPhysBench authors",
    "source_group": "visual-t2vphysbench:t2vphysbench-human-avg"
  },
  {
    "model": "pika-2-2",
    "bench": "t2vphysbench-human-avg",
    "score": 0.32,
    "source": "https://arxiv.org/html/2505.00337v1#S4.T2",
    "source_id": "visual-t2vphysbench",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Pika 2.2; Table 2 Avg. Score 0.32; three human annotators per video, four-level physical-law adherence mapped to [0,1]; Table 1: Pika 2.2 (Pika Labs, 2025).",
    "evidence": {
      "source_model_label": "Pika 2.2",
      "table": "S4.T2",
      "reported_value": "0.32",
      "divisor": 1,
      "measurement_origin": "explicit original human evaluation by the benchmark authors",
      "protocol": "Fully manual evaluation: three annotators review every generated video; four quality levels mapped to [0,1]; averaged across prompts and annotators.",
      "source_row": [
        "Pika 2.2",
        "0.38",
        "0.19",
        "0.40",
        "0.32"
      ],
      "source_header": [
        "Model",
        "Newton Principles",
        "Conservation Principles",
        "Phenomenon Principles",
        "Avg. Score"
      ],
      "model_info_row": [
        "Pika 2.2 [44]",
        "2025",
        "N/A",
        "Pika Labs",
        "No"
      ],
      "date_basis": "Report version date (v1, 2025-05-01), not an asserted generation date",
      "model_identity_scope": "Release identified by Table 1 (year, parameters, organisation, openness); no API snapshot claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2505.00337v1#S3.SS3.p1.1",
      "provenance_quote": "we adopt a fully manual evaluation protocol, following VideoPhy. Three annotators (undergraduate or graduate students) independently review every generated video ... We compare all the models listed in Table 1 and present the overall result in Table 2.",
      "evaluator_relationship": "academic benchmark authors evaluating other providers' models"
    },
    "evaluator": "T2VPhysBench authors",
    "source_group": "visual-t2vphysbench:t2vphysbench-human-avg"
  },
  {
    "model": "flux-2-dev",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8948,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "FLUX.2-dev; Table 2 English HCQ Total 89.48 (Chinese 88.60 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "FLUX.2-dev",
      "table": "Sx3.T2",
      "reported_value": "89.48",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "FLUX.2-dev",
        "89.48",
        "97.5",
        "78.6",
        "94.7",
        "89.4",
        "87.1",
        "88.60",
        "97.2",
        "74.4",
        "94.6",
        "89.6",
        "87.3"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "ernie-image",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8943,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "ERNIE-Image; Table 2 English HCQ Total 89.43 (Chinese 89.58 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "ERNIE-Image",
      "table": "Sx3.T2",
      "reported_value": "89.43",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "ERNIE-Image",
        "89.43",
        "97.2",
        "79.7",
        "95.0",
        "89.5",
        "85.7",
        "89.58",
        "97.5",
        "79.7",
        "95.4",
        "90.5",
        "84.8"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "z-image-base",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8811,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "Z-Image Base; Table 2 English HCQ Total 88.11 (Chinese 89.15 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "Z-Image Base",
      "table": "Sx3.T2",
      "reported_value": "88.11",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "Z-Image Base",
        "88.11",
        "95.5",
        "79.1",
        "93.6",
        "86.9",
        "85.3",
        "89.15",
        "96.7",
        "79.8",
        "94.3",
        "88.5",
        "86.4"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "qwen-image",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8772,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "Qwen-Image; Table 2 English HCQ Total 87.72 (Chinese 88.55 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "Qwen-Image",
      "table": "Sx3.T2",
      "reported_value": "87.72",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "Qwen-Image",
        "87.72",
        "95.7",
        "79.4",
        "93.4",
        "87.0",
        "83.1",
        "88.55",
        "96.3",
        "82.4",
        "93.6",
        "87.9",
        "82.6"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8708,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "FLUX.2-Klein-9B; Table 2 English HCQ Total 87.08 (Chinese 85.35 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "FLUX.2-Klein-9B",
      "table": "Sx3.T2",
      "reported_value": "87.08",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "FLUX.2-Klein-9B",
        "87.08",
        "95.8",
        "72.3",
        "94.0",
        "87.9",
        "85.3",
        "85.35",
        "96.7",
        "61.0",
        "94.4",
        "89.1",
        "85.5"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "hidream-o1-image",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.868,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "HiDream-O1; Table 2 English HCQ Total 86.80 (Chinese 87.00 not imported); official defaults, prompt enhancement disabled; HiDream-O1 = HiDream-O1-Image (bibliography: Cai et al. 2026, arXiv:2605.11061).",
    "evidence": {
      "source_model_label": "HiDream-O1",
      "table": "Sx3.T2",
      "reported_value": "86.80",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "HiDream-O1",
        "86.80",
        "95.3",
        "73.1",
        "92.7",
        "87.3",
        "85.5",
        "87.00",
        "95.1",
        "76.1",
        "92.2",
        "87.0",
        "84.6"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "z-image-turbo",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8567,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "Z-Image Turbo; Table 2 English HCQ Total 85.67 (Chinese 85.98 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "Z-Image Turbo",
      "table": "Sx3.T2",
      "reported_value": "85.67",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "Z-Image Turbo",
        "85.67",
        "93.9",
        "78.5",
        "91.6",
        "84.0",
        "80.4",
        "85.98",
        "95.1",
        "77.5",
        "91.4",
        "84.8",
        "81.1"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "flux-2-klein-4b",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8433,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "FLUX.2-Klein-4B; Table 2 English HCQ Total 84.33 (Chinese 82.29 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "FLUX.2-Klein-4B",
      "table": "Sx3.T2",
      "reported_value": "84.33",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "FLUX.2-Klein-4B",
        "84.33",
        "96.0",
        "61.9",
        "93.5",
        "86.8",
        "83.4",
        "82.29",
        "95.3",
        "52.0",
        "93.6",
        "88.3",
        "82.3"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "glm-image",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8394,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "GLM-Image; Table 2 English HCQ Total 83.94 (Chinese 84.88 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "GLM-Image",
      "table": "Sx3.T2",
      "reported_value": "83.94",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "GLM-Image",
        "83.94",
        "92.6",
        "73.4",
        "88.9",
        "82.9",
        "82.0",
        "84.88",
        "93.1",
        "76.2",
        "89.1",
        "83.9",
        "82.2"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "longcat-image",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.8391,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "LongCat-Image; Table 2 English HCQ Total 83.91 (Chinese 85.97 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "LongCat-Image",
      "table": "Sx3.T2",
      "reported_value": "83.91",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "LongCat-Image",
        "83.91",
        "96.2",
        "63.8",
        "91.6",
        "84.7",
        "83.3",
        "85.97",
        "96.2",
        "70.0",
        "92.8",
        "87.7",
        "83.2"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "flux-1-dev",
    "bench": "qc-t2i-bench-en-hcq",
    "score": 0.7621,
    "source": "https://arxiv.org/html/2608.24112v1#Sx3.T2",
    "source_id": "visual-qcbench",
    "by": "independent",
    "date": "2026-08-25",
    "note": "FLUX.1-dev; Table 2 English HCQ Total 76.21 (Chinese 32.75 not imported); official defaults, prompt enhancement disabled.",
    "evidence": {
      "source_model_label": "FLUX.1-dev",
      "table": "Sx3.T2",
      "reported_value": "76.21",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Official defaults, configurable prompt enhancement disabled; English HCQ Total over five capability views.",
      "source_row": [
        "FLUX.1-dev",
        "76.21",
        "89.9",
        "44.1",
        "88.8",
        "80.1",
        "78.2",
        "32.75",
        "23.0",
        "16.5",
        "47.2",
        "41.8",
        "35.3"
      ],
      "source_header": [
        "Model",
        "EN Total",
        "EN Ent.",
        "EN Text",
        "EN Attr.",
        "EN Rel.",
        "EN Sem.",
        "ZH Total",
        "ZH Ent.",
        "ZH Text",
        "ZH Attr.",
        "ZH Rel.",
        "ZH Sem."
      ],
      "date_basis": "Report version date (2026-08-25), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2608.24112v1#Sx4.SSx1.SSS0.Px1.p1.1",
      "provenance_quote": "We evaluate 13 recent open-source T2I systems ... We use official defaults and disable configurable prompt enhancement so every generator receives the same prompt.",
      "evaluator_relationship": "China Unicom benchmark authors evaluating other providers' models"
    },
    "evaluator": "QC-T2I-Bench authors (China Unicom)",
    "source_group": "visual-qcbench:qc-t2i-bench-en-hcq"
  },
  {
    "model": "gpt-image-2",
    "bench": "imagetime-prompt-only",
    "score": 0.786,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "GPT Image 2; Table 5 Overall mean 7.86/10 (C 7.86, D 7.87); 750 prompt-only cases; GPT-5.5 judge; API model page cited; quality setting not stated.",
    "evidence": {
      "source_model_label": "GPT Image 2 [48, 49]",
      "table": "S5.T5",
      "reported_value": "7.86",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "GPT Image 2 [48, 49]",
        "750",
        "7.86",
        "7.87",
        "7.86"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "nano-banana-2",
    "bench": "imagetime-prompt-only",
    "score": 0.745,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Nano Banana 2; Table 5 Overall mean 7.45/10 (C 7.43, D 7.47); 750 prompt-only cases; GPT-5.5 judge; cited Gemini 3.1 Flash Image model card.",
    "evidence": {
      "source_model_label": "Nano Banana 2 [50]",
      "table": "S5.T5",
      "reported_value": "7.45",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "Nano Banana 2 [50]",
        "750",
        "7.43",
        "7.47",
        "7.45"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "imagetime-prompt-only",
    "score": 0.716,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Seedream 5.0 Lite; Table 5 Overall mean 7.16/10 (C 7.13, D 7.20); 750 prompt-only cases; GPT-5.5 judge.",
    "evidence": {
      "source_model_label": "Seedream 5.0 Lite [51]",
      "table": "S5.T5",
      "reported_value": "7.16",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "Seedream 5.0 Lite [51]",
        "750",
        "7.13",
        "7.20",
        "7.16"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "flux-2-pro",
    "bench": "imagetime-prompt-only",
    "score": 0.61,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "FLUX.2 Pro; Table 5 Overall mean 6.10/10 (C 5.92, D 6.28); 750 prompt-only cases; GPT-5.5 judge.",
    "evidence": {
      "source_model_label": "FLUX.2 Pro [47]",
      "table": "S5.T5",
      "reported_value": "6.10",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "FLUX.2 Pro [47]",
        "750",
        "5.92",
        "6.28",
        "6.10"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "z-image-turbo",
    "bench": "imagetime-prompt-only",
    "score": 0.541,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Z-Image-Turbo; Table 5 Overall mean 5.41/10 (C 5.14, D 5.69); 750 prompt-only cases; GPT-5.5 judge.",
    "evidence": {
      "source_model_label": "Z-Image-Turbo [55]",
      "table": "S5.T5",
      "reported_value": "5.41",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "Z-Image-Turbo [55]",
        "750",
        "5.14",
        "5.69",
        "5.41"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "qwen-image-2512",
    "bench": "imagetime-prompt-only",
    "score": 0.532,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "Qwen-Image-2512; Table 5 Overall mean 5.32/10 (C 5.09, D 5.55); 750 prompt-only cases; GPT-5.5 judge.",
    "evidence": {
      "source_model_label": "Qwen-Image-2512 [52, 53]",
      "table": "S5.T5",
      "reported_value": "5.32",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "Qwen-Image-2512 [52, 53]",
        "750",
        "5.09",
        "5.55",
        "5.32"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "hunyuanimage-2-1",
    "bench": "imagetime-prompt-only",
    "score": 0.498,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "HunyuanImage-2.1; Table 5 Overall mean 4.98/10 (C 4.91, D 5.04); 750 prompt-only cases; GPT-5.5 judge.",
    "evidence": {
      "source_model_label": "HunyuanImage-2.1 [54]",
      "table": "S5.T5",
      "reported_value": "4.98",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "HunyuanImage-2.1 [54]",
        "750",
        "4.91",
        "5.04",
        "4.98"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "sdxl",
    "bench": "imagetime-prompt-only",
    "score": 0.157,
    "source": "https://arxiv.org/html/2606.10620v1#S5.T5",
    "source_id": "visual-imagetime",
    "by": "independent",
    "date": "2026-06-09",
    "note": "SDXL; Table 5 Overall mean 1.57/10 (C 1.49, D 1.64); 750 prompt-only cases; GPT-5.5 judge; SDXL paper cited.",
    "evidence": {
      "source_model_label": "SDXL [10]",
      "table": "S5.T5",
      "reported_value": "1.57",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Prompt-only 2x2 motion sheet; same 750 prompts; GPT-5.5 structured judge, 0–10 integer C/D scores; Overall = balanced average of C and D means.",
      "source_row": [
        "SDXL [10]",
        "750",
        "1.49",
        "1.64",
        "1.57"
      ],
      "source_header": [
        "Model",
        "N",
        "C mean",
        "D mean",
        "Overall mean"
      ],
      "date_basis": "Report version date (2026-06-09), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.10620v1#S5.SS1.p1.1",
      "provenance_quote": "We evaluate eight text-to-image models in the prompt-only ImageTime setting ... Each model is evaluated on the same 750 ImageTime prompts. (S5.1) In all reported experiments, GPT-5.5 is used to score the images generated by every evaluated model. (S4.2)",
      "evaluator_relationship": "benchmark authors (affiliations not stated) evaluating other providers' models"
    },
    "evaluator": "ImageTime authors (Xinrui Wu, Lichen Huang)",
    "source_group": "visual-imagetime:imagetime-prompt-only"
  },
  {
    "model": "nano-banana-pro",
    "bench": "gebench-en-ge-score",
    "score": 0.612,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Nano Banana pro; Table 1 English GE-Score 61.20 (Chinese 69.62 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "Nano Banana pro (Google, 2025b)",
      "table": "S3.T1",
      "reported_value": "61.20",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Nano Banana pro (Google, 2025b)",
        "84.50",
        "68.65",
        "65.75",
        "64.35",
        "64.83",
        "69.62",
        "84.32",
        "69.51",
        "46.33",
        "47.20",
        "58.64",
        "61.20"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "nano-banana",
    "bench": "gebench-en-ge-score",
    "score": 0.5212,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Nano Banana; Table 1 English GE-Score 52.12 (Chinese 56.74 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "Nano Banana (Google, 2025b)",
      "table": "S3.T1",
      "reported_value": "52.12",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Nano Banana (Google, 2025b)",
        "64.36",
        "34.16",
        "64.82",
        "65.89",
        "54.48",
        "56.74",
        "64.80",
        "50.75",
        "48.88",
        "47.12",
        "49.04",
        "52.12"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "gebench-en-ge-score",
    "score": 0.6316,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "GPT-image-1.5; Table 1 English GE-Score 63.16 (Chinese 63.22 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "GPT-image-1.5 (OpenAI, 2025)",
      "table": "S3.T1",
      "reported_value": "63.16",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "GPT-image-1.5 (OpenAI, 2025)",
        "83.79",
        "56.97",
        "60.11",
        "55.65",
        "53.33",
        "63.22",
        "80.80",
        "58.87",
        "63.68",
        "58.93",
        "49.23",
        "63.16"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "gpt-image-1",
    "bench": "gebench-en-ge-score",
    "score": 0.5564,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "GPT-image-1.0; Table 1 English GE-Score 55.64 (Chinese 52.39 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "GPT-image-1.0 (OpenAI, 2025)",
      "table": "S3.T1",
      "reported_value": "55.64",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "GPT-image-1.0 (OpenAI, 2025)",
        "64.72",
        "49.20",
        "57.31",
        "59.04",
        "31.68",
        "52.39",
        "60.92",
        "64.33",
        "58.94",
        "56.16",
        "37.84",
        "55.64"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "seedream-4-5",
    "bench": "gebench-en-ge-score",
    "score": 0.5001,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Seedream 4.5; Table 1 English GE-Score 50.01 (Chinese 55.91 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "Seedream 4.5 (Seed, 2025)",
      "table": "S3.T1",
      "reported_value": "50.01",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Seedream 4.5 (Seed, 2025)",
        "63.64",
        "53.11",
        "56.48",
        "53.44",
        "52.90",
        "55.91",
        "49.49",
        "45.30",
        "53.81",
        "51.80",
        "49.63",
        "50.01"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "seedream-4-0",
    "bench": "gebench-en-ge-score",
    "score": 0.4646,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Seedream 4.0; Table 1 English GE-Score 46.46 (Chinese 52.88 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "Seedream 4.0 (Seedream et al., 2025)",
      "table": "S3.T1",
      "reported_value": "46.46",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Seedream 4.0 (Seedream et al., 2025)",
        "62.04",
        "48.64",
        "49.28",
        "50.93",
        "53.53",
        "52.88",
        "53.28",
        "37.57",
        "47.92",
        "49.36",
        "44.17",
        "46.46"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "flux-2-pro",
    "bench": "gebench-en-ge-score",
    "score": 0.5118,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Flux-2-pro; Table 1 English GE-Score 51.18 (Chinese 57.54 not imported); official default configuration.",
    "evidence": {
      "source_model_label": "Flux-2-pro (Labs, 2025)",
      "table": "S3.T1",
      "reported_value": "51.18",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Flux-2-pro (Labs, 2025)",
        "68.83",
        "55.07",
        "58.13",
        "55.41",
        "50.24",
        "57.54",
        "61.00",
        "52.17",
        "49.92",
        "47.16",
        "45.67",
        "51.18"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "bagel",
    "bench": "gebench-en-ge-score",
    "score": 0.28,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Bagel; Table 1 English GE-Score 28.00 (Chinese 28.85 not imported); official default configuration; thinking mode not stated.",
    "evidence": {
      "source_model_label": "Bagel (Deng et al., 2025)",
      "table": "S3.T1",
      "reported_value": "28.00",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Bagel (Deng et al., 2025)",
        "34.84",
        "13.45",
        "27.36",
        "33.52",
        "35.10",
        "28.85",
        "32.91",
        "8.61",
        "26.08",
        "35.12",
        "37.30",
        "28.00"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "qwen-image-edit",
    "bench": "gebench-en-ge-score",
    "score": 0.3301,
    "source": "https://arxiv.org/html/2602.09007v2#S3.T1",
    "source_id": "visual-gebench",
    "by": "independent",
    "date": "2026-02-10",
    "note": "Qwen-Image-Edit; Table 1 English GE-Score 33.01 (Chinese 33.72 not imported); official default configuration; label without 2509/2511 suffix = original release.",
    "evidence": {
      "source_model_label": "Qwen-Image-Edit (Wu et al., 2025)",
      "table": "S3.T1",
      "reported_value": "33.01",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "GUI next-state generation; official default configurations; three VLM judges; English GE-Score.",
      "source_row": [
        "Qwen-Image-Edit (Wu et al., 2025)",
        "41.12",
        "26.79",
        "23.78",
        "26.10",
        "50.80",
        "33.72",
        "40.12",
        "18.61",
        "25.80",
        "25.95",
        "54.55",
        "33.01"
      ],
      "source_header": [
        "Model",
        "ZH single-step",
        "ZH multi-step",
        "ZH fiction-app",
        "ZH real-app",
        "ZH grounding",
        "ZH GE Score",
        "EN single-step",
        "EN multi-step",
        "EN fiction-app",
        "EN real-app",
        "EN grounding",
        "EN GE Score"
      ],
      "date_basis": "Report version date (2026-02-10), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2602.09007v2#S4.SS1.SSS0.Px2.p1.1",
      "provenance_quote": "Our experimental evaluation encompasses 12 image generation models ... we use official default configurations for evaluated models and perform evaluation three times for each generated GUIs trajectory.",
      "evaluator_relationship": "StepFun-led benchmark authors evaluating other providers' models (no StepFun model in the table)"
    },
    "evaluator": "GEBench authors (StepFun et al.)",
    "source_group": "visual-gebench:gebench-en-ge-score"
  },
  {
    "model": "gpt-image-1",
    "bench": "imagenworld-tig-human",
    "score": 0.91,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "GPT-Image-1; Table 3 Text-guided Image Generation (TIG) human overall 0.91±0.10 (mean±sd across images); default configuration; closed model via API.",
    "evidence": {
      "source_model_label": "GPT-Image-1",
      "table": "S5.T3",
      "reported_value": "0.91",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "GPT-Image-1",
        "0.90±\\pm0.14",
        "0.92±\\pm0.10",
        "0.91±\\pm0.13",
        "0.92±\\pm0.12",
        "0.91±\\pm0.10",
        "0.92±\\pm0.13",
        "0.52",
        "0.39"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "qwen-image",
    "bench": "imagenworld-tig-human",
    "score": 0.87,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "Qwen-Image; Table 3 Text-guided Image Generation (TIG) human overall 0.87±0.15 (mean±sd across images); default configuration.",
    "evidence": {
      "source_model_label": "Qwen-Image",
      "table": "S5.T3",
      "reported_value": "0.87",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "Qwen-Image",
        "0.82±\\pm0.22",
        "0.92±\\pm0.11",
        "0.88±\\pm0.17",
        "0.85±\\pm0.22",
        "0.87±\\pm0.15",
        "0.86±\\pm0.20",
        "0.75",
        "0.76"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "flux-1-krea-dev",
    "bench": "imagenworld-tig-human",
    "score": 0.82,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "Flux.1-Krea-dev; Table 3 Text-guided Image Generation (TIG) human overall 0.82±0.17 (mean±sd across images); default configuration.",
    "evidence": {
      "source_model_label": "Flux.1-Krea-dev",
      "table": "S5.T3",
      "reported_value": "0.82",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "Flux.1-Krea-dev",
        "0.77±\\pm0.23",
        "0.85±\\pm0.16",
        "0.84±\\pm0.18",
        "0.80±\\pm0.22",
        "0.82±\\pm0.17",
        "0.82±\\pm0.23",
        "0.67",
        "0.79"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "bagel",
    "bench": "imagenworld-tig-human",
    "score": 0.76,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "BAGEL; Table 3 Text-guided Image Generation (TIG) human overall 0.76±0.21 (mean±sd across images); default configuration; thinking mode not stated.",
    "evidence": {
      "source_model_label": "BAGEL",
      "table": "S5.T3",
      "reported_value": "0.76",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "BAGEL",
        "0.72±\\pm0.24",
        "0.80±\\pm0.20",
        "0.82±\\pm0.21",
        "0.73±\\pm0.29",
        "0.76±\\pm0.21",
        "0.75±\\pm0.26",
        "0.74",
        "0.78"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "omnigen2",
    "bench": "imagenworld-tig-human",
    "score": 0.74,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "OmniGen2; Table 3 Text-guided Image Generation (TIG) human overall 0.74±0.23 (mean±sd across images); default configuration.",
    "evidence": {
      "source_model_label": "OmniGen2",
      "table": "S5.T3",
      "reported_value": "0.74",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "OmniGen2",
        "0.67±\\pm0.25",
        "0.77±\\pm0.23",
        "0.78±\\pm0.24",
        "0.72±\\pm0.32",
        "0.74±\\pm0.23",
        "0.75±\\pm0.27",
        "0.71",
        "0.80"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "sdxl",
    "bench": "imagenworld-tig-human",
    "score": 0.64,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "SDXL; Table 3 Text-guided Image Generation (TIG) human overall 0.64±0.25 (mean±sd across images); default configuration; SDXL (Podell et al., 2023).",
    "evidence": {
      "source_model_label": "SDXL",
      "table": "S5.T3",
      "reported_value": "0.64",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Generation (TIG); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "SDXL",
        "0.55±\\pm0.29",
        "0.70±\\pm0.25",
        "0.70±\\pm0.27",
        "0.61±\\pm0.31",
        "0.64±\\pm0.25",
        "0.65±\\pm0.30",
        "0.75",
        "0.79"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Generation (TIG)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tig-human"
  },
  {
    "model": "gpt-image-1",
    "bench": "imagenworld-tie-human",
    "score": 0.82,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "GPT-Image-1; Table 3 Text-guided Image Editing (TIE) human overall 0.82±0.15 (mean±sd across images); default configuration; closed model via API.",
    "evidence": {
      "source_model_label": "GPT-Image-1",
      "table": "S5.T3",
      "reported_value": "0.82",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Editing (TIE); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "GPT-Image-1",
        "0.77±\\pm0.17",
        "0.86±\\pm0.15",
        "0.86±\\pm0.17",
        "0.79±\\pm0.22",
        "0.82±\\pm0.15",
        "0.80±\\pm0.28",
        "0.65",
        "0.58"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Editing (TIE)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tie-human"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "imagenworld-tie-human",
    "score": 0.7,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "Flux.1-Kontext-dev; Table 3 Text-guided Image Editing (TIE) human overall 0.70±0.21 (mean±sd across images); default configuration.",
    "evidence": {
      "source_model_label": "Flux.1-Kontext-dev",
      "table": "S5.T3",
      "reported_value": "0.70",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Editing (TIE); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "Flux.1-Kontext-dev",
        "0.52±\\pm0.31",
        "0.76±\\pm0.22",
        "0.75±\\pm0.22",
        "0.76±\\pm0.26",
        "0.70±\\pm0.21",
        "0.63±\\pm0.32",
        "0.68",
        "0.71"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Editing (TIE)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tie-human"
  },
  {
    "model": "bagel",
    "bench": "imagenworld-tie-human",
    "score": 0.67,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "BAGEL; Table 3 Text-guided Image Editing (TIE) human overall 0.67±0.22 (mean±sd across images); default configuration; thinking mode not stated.",
    "evidence": {
      "source_model_label": "BAGEL",
      "table": "S5.T3",
      "reported_value": "0.67",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Editing (TIE); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "BAGEL",
        "0.59±\\pm0.28",
        "0.64±\\pm0.26",
        "0.73±\\pm0.23",
        "0.70±\\pm0.25",
        "0.67±\\pm0.22",
        "0.59±\\pm0.33",
        "0.62",
        "0.66"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Editing (TIE)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tie-human"
  },
  {
    "model": "omnigen2",
    "bench": "imagenworld-tie-human",
    "score": 0.58,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "OmniGen2; Table 3 Text-guided Image Editing (TIE) human overall 0.58±0.27 (mean±sd across images); default configuration.",
    "evidence": {
      "source_model_label": "OmniGen2",
      "table": "S5.T3",
      "reported_value": "0.58",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Editing (TIE); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "OmniGen2",
        "0.38±\\pm0.31",
        "0.63±\\pm0.29",
        "0.67±\\pm0.30",
        "0.66±\\pm0.31",
        "0.58±\\pm0.27",
        "0.57±\\pm0.32",
        "0.77",
        "0.72"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Editing (TIE)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint; no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tie-human"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.573529,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "anthropic/opus_46",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.415441,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'DeepSeek-v3.2 (Think)', model_config 'deepseek/deepseek_v32_dec' (deepseek-reasoner (DeepSeek API, V3.2 thinking, Dec 2025; HF deepseek-ai/DeepSeek-V3.2), per configs/models/deepseek/deepseek_v32_dec.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 16 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "deepseek/deepseek_v32_dec",
      "problems": 17,
      "attempts_min": 16,
      "attempts_max": 16
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.661765,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "gemini/gemini-31-pro",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.433824,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Gemini 3 Flash', model_config 'gemini/gemini-3-flash' (gemini-3-flash-preview, thinking_level high, per configs/models/gemini/gemini-3-flash.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 16 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "gemini/gemini-3-flash",
      "problems": 17,
      "attempts_min": 16,
      "attempts_max": 16
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.511029,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Gemini 3 Pro (preview)', model_config 'gemini/riftrunner' (gemini-3-pro-preview, thinking included (default level), per configs/models/gemini/riftrunner.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 16 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "gemini/riftrunner",
      "problems": 17,
      "attempts_min": 16,
      "attempts_max": 16
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.382353,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "glm/glm-5",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-2",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.522059,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'GPT-5.2 (high)', model_config 'openai/gpt-52-high' (gpt-5.2, reasoning effort high, per configs/models/openai/gpt-52-high.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 16 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "openai/gpt-52-high",
      "problems": 17,
      "attempts_min": 16,
      "attempts_max": 16
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.602941,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "openai/gpt-54",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.5,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Grok 4.1 Fast (Reasoning)', model_config 'xai/grok-41-fast-reasoning' (grok-4-1-fast-reasoning (xAI API), per configs/models/xai/grok-41-fast-reasoning.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 16 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "xai/grok-41-fast-reasoning",
      "problems": 17,
      "attempts_min": 16,
      "attempts_max": 16
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-5",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.419118,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Kimi K2.5 (Think)', model_config 'moonshot/k25' (moonshotai/kimi-k2.5 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k25.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "moonshot/k25",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.338235,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'NVIDIA-Nemotron-3-Super', model_config 'nvidia/nemotron-3-super' (nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 on vLLM, thinking enabled, per configs/models/nvidia/nemotron-3-super.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "nvidia/nemotron-3-super",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.279412,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3-30B-A3B-2507-Think', model_config 'qwen/qwen3_30b_a3b_2507' (Qwen/Qwen3-30B-A3B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_30b_a3b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3_30b_a3b_2507",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.323529,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3-4B-2507-Think', model_config 'qwen/qwen3_4b_2507' (Qwen/Qwen3-4B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_4b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3_4b_2507",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.411765,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3.5-27B', model_config 'qwen/qwen3.5_27b' (Qwen3.5-27B self-hosted vLLM, default thinking, temperature 1.0, per configs/models/qwen/qwen3.5_27b.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3.5_27b",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.397059,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3.5-35B-A3B', model_config 'qwen/qwen3.5_35b_a3b' (Qwen/Qwen3.5-35B-A3B on vLLM, default thinking, per configs/models/qwen/qwen3.5_35b_a3b.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3.5_35b_a3b",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.382353,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3.5-397b-a17b', model_config 'qwen/qwen3.5_397b' (qwen/qwen3.5-397b-a17b via OpenRouter pinned to Alibaba, per configs/models/qwen/qwen3.5_397b.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3.5_397b",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.397059,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Qwen3.5-9B', model_config 'qwen/qwen3.5_9b' (Qwen/Qwen3.5-9B via Together, default thinking, per configs/models/qwen/qwen3.5_9b.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "qwen/qwen3.5_9b",
      "problems": 17,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.419118,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 17/17 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-1225_outputs",
      "revision": "6e427c4860254adc93ee95ebd37dc2b6c6028b4a",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "47f8c87d00d039f6ee3e58bf36711a7cba93cb04e6c094814a2ce6555f48044b",
      "model_config": "stepfun/3.5-flash",
      "problems": 17,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.728261,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "anthropic/opus_46",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.521739,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "anthropic/opus_47",
      "problems": 23,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.570652,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'DeepSeek-v3.2 (Think)', model_config 'deepseek/deepseek_v32_dec' (deepseek-reasoner (DeepSeek API, V3.2 thinking, Dec 2025; HF deepseek-ai/DeepSeek-V3.2), per configs/models/deepseek/deepseek_v32_dec.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "deepseek/deepseek_v32_dec",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.706522,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "gemini/gemini-31-pro",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.581522,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Gemini 3 Flash', model_config 'gemini/gemini-3-flash' (gemini-3-flash-preview, thinking_level high, per configs/models/gemini/gemini-3-flash.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "gemini/gemini-3-flash",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.619565,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Gemini 3 Pro (preview)', model_config 'gemini/riftrunner' (gemini-3-pro-preview, thinking included (default level), per configs/models/gemini/riftrunner.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "gemini/riftrunner",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.543478,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "glm/glm-5",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.652174,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "glm/glm-51",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-2",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.679348,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'GPT-5.2 (high)', model_config 'openai/gpt-52-high' (gpt-5.2, reasoning effort high, per configs/models/openai/gpt-52-high.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "openai/gpt-52-high",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.76087,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "openai/gpt-54",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.73913,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "openai/gpt-55",
      "problems": 23,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.532609,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Grok 4.1 Fast (Reasoning)', model_config 'xai/grok-41-fast-reasoning' (grok-4-1-fast-reasoning (xAI API), per configs/models/xai/grok-41-fast-reasoning.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "xai/grok-41-fast-reasoning",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-5",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Kimi K2.5 (Think)', model_config 'moonshot/k25' (moonshotai/kimi-k2.5 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k25.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "moonshot/k25",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-6",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.717391,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Kimi K2.6 (Think)', model_config 'moonshot/k26' (moonshotai/kimi-k2.6 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k26.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "moonshot/k26",
      "problems": 23,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.48913,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'NVIDIA-Nemotron-3-Super', model_config 'nvidia/nemotron-3-super' (nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 on vLLM, thinking enabled, per configs/models/nvidia/nemotron-3-super.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "nvidia/nemotron-3-super",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.391304,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3-30B-A3B-2507-Think', model_config 'qwen/qwen3_30b_a3b_2507' (Qwen/Qwen3-30B-A3B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_30b_a3b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3_30b_a3b_2507",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.23913,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3-4B-2507-Think', model_config 'qwen/qwen3_4b_2507' (Qwen/Qwen3-4B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_4b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3_4b_2507",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.532609,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3.5-27B', model_config 'qwen/qwen3.5_27b' (Qwen3.5-27B self-hosted vLLM, default thinking, temperature 1.0, per configs/models/qwen/qwen3.5_27b.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3.5_27b",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.5,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3.5-35B-A3B', model_config 'qwen/qwen3.5_35b_a3b' (Qwen/Qwen3.5-35B-A3B on vLLM, default thinking, per configs/models/qwen/qwen3.5_35b_a3b.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3.5_35b_a3b",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.548913,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3.5-397b-a17b', model_config 'qwen/qwen3.5_397b' (qwen/qwen3.5-397b-a17b via OpenRouter pinned to Alibaba, per configs/models/qwen/qwen3.5_397b.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3.5_397b",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.445652,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Qwen3.5-9B', model_config 'qwen/qwen3.5_9b' (Qwen/Qwen3.5-9B via Together, default thinking, per configs/models/qwen/qwen3.5_9b.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "qwen/qwen3.5_9b",
      "problems": 23,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.603261,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 8 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "stepfun/3.5-flash",
      "problems": 23,
      "attempts_min": 8,
      "attempts_max": 8
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.40625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "anthropic/opus_46",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.40625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "anthropic/opus_47",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.601562,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "anthropic/opus_48_max",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "gemini/gemini-31-pro",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.507812,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "gemini/gemini-35-flash",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.414062,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "glm/glm-5",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.390625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "glm/glm-51",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.460938,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "glm/glm-52",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-2",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GPT-5.2 (high)', model_config 'openai/gpt-52-high' (gpt-5.2, reasoning effort high, per configs/models/openai/gpt-52-high.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "openai/gpt-52-high",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.59375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "openai/gpt-54",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.734375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "openai/gpt-55",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.320312,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Grok 4.1 Fast (Reasoning)', model_config 'xai/grok-41-fast-reasoning' (grok-4-1-fast-reasoning (xAI API), per configs/models/xai/grok-41-fast-reasoning.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "xai/grok-41-fast-reasoning",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-6",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.429688,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Kimi K2.6 (Think)', model_config 'moonshot/k26' (moonshotai/kimi-k2.6 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k26.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "moonshot/k26",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.304688,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'NVIDIA-Nemotron-3-Super', model_config 'nvidia/nemotron-3-super' (nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 on vLLM, thinking enabled, per configs/models/nvidia/nemotron-3-super.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "nvidia/nemotron-3-super",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.226562,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Qwen3-30B-A3B-2507-Think', model_config 'qwen/qwen3_30b_a3b_2507' (Qwen/Qwen3-30B-A3B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_30b_a3b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "qwen/qwen3_30b_a3b_2507",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.179688,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Qwen3-4B-2507-Think', model_config 'qwen/qwen3_4b_2507' (Qwen/Qwen3-4B-Thinking-2507 on vLLM, per configs/models/qwen/qwen3_4b_2507.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "qwen/qwen3_4b_2507",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.3125,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Qwen3.5-27B', model_config 'qwen/qwen3.5_27b' (Qwen3.5-27B self-hosted vLLM, default thinking, temperature 1.0, per configs/models/qwen/qwen3.5_27b.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 3-4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "qwen/qwen3.5_27b",
      "problems": 32,
      "attempts_min": 3,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.304688,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Qwen3.5-35B-A3B', model_config 'qwen/qwen3.5_35b_a3b' (Qwen/Qwen3.5-35B-A3B on vLLM, default thinking, per configs/models/qwen/qwen3.5_35b_a3b.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "qwen/qwen3.5_35b_a3b",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.273438,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Qwen3.5-9B', model_config 'qwen/qwen3.5_9b' (Qwen/Qwen3.5-9B via Together, default thinking, per configs/models/qwen/qwen3.5_9b.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "qwen/qwen3.5_9b",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.328125,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "stepfun/3.5-flash",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.320312,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "stepfun/3.7-flash",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "anthropic/opus_46",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.508333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "anthropic/opus_47",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.75,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "anthropic/opus_48_max",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.683333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "gemini/gemini-31-pro",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.558333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "gemini/gemini-35-flash",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.458333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "glm/glm-5",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.491667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "glm/glm-51",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.616667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "glm/glm-52",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.677778,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 2-4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "openai/gpt-54",
      "problems": 30,
      "attempts_min": 2,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.775,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "openai/gpt-55",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-6",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.558333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Kimi K2.6 (Think)', model_config 'moonshot/k26' (moonshotai/kimi-k2.6 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k26.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "moonshot/k26",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.433333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "stepfun/3.5-flash",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.45,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "stepfun/3.7-flash",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.725,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "anthropic/fable_5_max",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.775,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Claude-Fable-5.1 (max)', model_config 'anthropic/fable_51_max' (claude-fable-5-1, adaptive thinking, effort max, per configs/models/anthropic/fable_51_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "anthropic/fable_51_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.575,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "anthropic/opus_47",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "anthropic/opus_48_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.6375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "gemini/gemini-31-pro",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.525,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "gemini/gemini-35-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.525,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Gemini 3.6 Flash', model_config 'gemini/gemini-36-flash' (gemini-3.6-flash (Google AI Studio via INSAIT proxy), high, per configs/models/gemini/gemini-36-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "gemini/gemini-36-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.575,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Gemini 3.7 Flash', model_config 'gemini/gemini-37-flash' (gemini-3.7-flash, reasoning_effort high, per configs/models/gemini/gemini-37-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "gemini/gemini-37-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.641667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (gemini-3.8-flash, reasoning_effort high, per configs/models/gemini/gemini-38-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "gemini/gemini-38-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.3625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "glm/glm-51",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.475,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "glm/glm-52",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.6875,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "openai/gpt-55",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.908333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (gpt-6-astra, reasoning_effort max, per configs/models/openai/gpt-6-astra.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "openai/gpt-6-astra",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.541667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (moonshotai/kimi-k3 via OpenRouter pinned to Moonshot AI, reasoning max, per configs/models/moonshot/k3.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "moonshot/k3",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.01875,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Qwen3.5-2B', model_config 'qwen/qwen3.5_2b' (Qwen3.5-2B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.5_2b.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "qwen/qwen3.5_2b",
      "problems": 40,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.3375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Qwen3.6-35B', model_config 'qwen/qwen3.6_35b' (Qwen/Qwen3.6-35B-A3B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.6_35b.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "qwen/qwen3.6_35b",
      "problems": 40,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.591667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (qwen/qwen3.8-max via OpenRouter pinned to Alibaba (HF id Qwen/Qwen3.8-2.4T-A95B), reasoning max, per configs/models/qwen/qwen3.8_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "qwen/qwen3.8_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.3375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "stepfun/3.5-flash",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.358333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "stepfun/3.7-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.875,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "anthropic/fable_5_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.875,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Claude-Fable-5.1 (max)', model_config 'anthropic/fable_51_max' (claude-fable-5-1, adaptive thinking, effort max, per configs/models/anthropic/fable_51_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "anthropic/fable_51_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.65,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "anthropic/opus_48_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.516667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "gemini/gemini-31-pro",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.441667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "gemini/gemini-35-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.541667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Gemini 3.6 Flash', model_config 'gemini/gemini-36-flash' (gemini-3.6-flash (Google AI Studio via INSAIT proxy), high, per configs/models/gemini/gemini-36-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "gemini/gemini-36-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.475,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Gemini 3.7 Flash', model_config 'gemini/gemini-37-flash' (gemini-3.7-flash, reasoning_effort high, per configs/models/gemini/gemini-37-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "gemini/gemini-37-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.583333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (gemini-3.8-flash, reasoning_effort high, per configs/models/gemini/gemini-38-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "gemini/gemini-38-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.525,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "glm/glm-51",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.566667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "glm/glm-52",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.775,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "openai/gpt-55",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.95,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (gpt-6-astra, reasoning_effort max, per configs/models/openai/gpt-6-astra.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "openai/gpt-6-astra",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.616667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (moonshotai/kimi-k3 via OpenRouter pinned to Moonshot AI, reasoning max, per configs/models/moonshot/k3.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "moonshot/k3",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.10625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Qwen3.5-2B', model_config 'qwen/qwen3.5_2b' (Qwen3.5-2B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.5_2b.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "qwen/qwen3.5_2b",
      "problems": 40,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Qwen3.6-35B', model_config 'qwen/qwen3.6_35b' (Qwen/Qwen3.6-35B-A3B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.6_35b.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "qwen/qwen3.6_35b",
      "problems": 40,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.691667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (qwen/qwen3.8-max via OpenRouter pinned to Alibaba (HF id Qwen/Qwen3.8-2.4T-A95B), reasoning max, per configs/models/qwen/qwen3.8_max.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "qwen/qwen3.8_max",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.466667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "stepfun/3.7-flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.854167,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "anthropic/fable_5_max",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.909722,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Claude-Fable-5.1 (max)', model_config 'anthropic/fable_51_max' (claude-fable-5-1, adaptive thinking, effort max, per configs/models/anthropic/fable_51_max.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "anthropic/fable_51_max",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.699653,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3-4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "anthropic/opus_48_max",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-5",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.826389,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Claude-Opus-5 (max)', model_config 'anthropic/opus_5_max' (claude-opus-5, adaptive thinking, effort max, per configs/models/anthropic/opus_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "anthropic/opus_5_max",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.666667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "gemini/gemini-31-pro",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.520833,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "gemini/gemini-35-flash",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.569444,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Gemini 3.6 Flash', model_config 'gemini/gemini-36-flash' (gemini-3.6-flash (Google AI Studio via INSAIT proxy), high, per configs/models/gemini/gemini-36-flash.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "gemini/gemini-36-flash",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Gemini 3.7 Flash', model_config 'gemini/gemini-37-flash' (gemini-3.7-flash, reasoning_effort high, per configs/models/gemini/gemini-37-flash.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "gemini/gemini-37-flash",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.673611,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (gemini-3.8-flash, reasoning_effort high, per configs/models/gemini/gemini-38-flash.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "gemini/gemini-38-flash",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.465278,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "glm/glm-52",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.83631,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 7 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "openai/gpt-55",
      "problems": 48,
      "attempts_min": 7,
      "attempts_max": 7
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.885417,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'GPT-5.6-Sol (max)', model_config 'openai/gpt-56-sol' (gpt-5.6-sol, reasoning effort max, per configs/models/openai/gpt-56-sol.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "openai/gpt-56-sol",
      "problems": 48,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.944444,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (gpt-6-astra, reasoning_effort max, per configs/models/openai/gpt-6-astra.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "openai/gpt-6-astra",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "grok-4-5",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.666667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Grok 4.5', model_config 'xai/grok-45' (grok-4.5 (xAI API), reasoning_effort high, per configs/models/xai/grok-45.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "xai/grok-45",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.715278,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (moonshotai/kimi-k3 via OpenRouter pinned to Moonshot AI, reasoning max, per configs/models/moonshot/k3.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "moonshot/k3",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "muse-spark-1-1",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.569444,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Muse Spark 1.1', model_config 'meta/muse_spark_11' (meta/muse-spark-1.1 via OpenRouter, per configs/models/meta/muse_spark_11.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "meta/muse_spark_11",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.041667,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Qwen3.5-2B', model_config 'qwen/qwen3.5_2b' (Qwen3.5-2B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.5_2b.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "qwen/qwen3.5_2b",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.270833,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Qwen3.6-35B', model_config 'qwen/qwen3.6_35b' (Qwen/Qwen3.6-35B-A3B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.6_35b.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "qwen/qwen3.6_35b",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.756944,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (qwen/qwen3.8-max via OpenRouter pinned to Alibaba (HF id Qwen/Qwen3.8-2.4T-A95B), reasoning max, per configs/models/qwen/qwen3.8_max.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "qwen/qwen3.8_max",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivmath-2026-06",
    "score": 0.305556,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0626_outputs/blob/f782eef32107837304e2ecd4e78b4cb60ee1e122/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-07",
    "note": "MathArena ArXivMath June 2026 outputs dataset (HF revision f782eef32107837304e2ecd4e78b4cb60ee1e122); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 48/48 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0626_outputs",
      "revision": "f782eef32107837304e2ecd4e78b4cb60ee1e122",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "5bfdd998c20c223ed272092564afa84a3a4f526f125b2366915632772d4534c6",
      "model_config": "stepfun/3.7-flash",
      "problems": 48,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.032258,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "anthropic/opus_46",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.040323,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "anthropic/opus_47",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.354839,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "anthropic/opus_48_max",
      "problems": 31,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.185484,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "gemini/gemini-31-pro",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.064516,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "gemini/gemini-35-flash",
      "problems": 31,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.116935,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "glm/glm-5",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.092742,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "glm/glm-51",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.104839,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "glm/glm-52",
      "problems": 31,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-2",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.258065,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GPT-5.2 (xhigh)', model_config 'openai/gpt-52' (gpt-5.2, reasoning effort xhigh, per configs/models/openai/gpt-52.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "openai/gpt-52",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.379032,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 3-4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "openai/gpt-54",
      "problems": 31,
      "attempts_min": 3,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.681452,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "openai/gpt-55",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-5",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.028226,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Kimi K2.5 (Think)', model_config 'moonshot/k25' (moonshotai/kimi-k2.5 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k25.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "moonshot/k25",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-6",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.116935,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Kimi K2.6 (Think)', model_config 'moonshot/k26' (moonshotai/kimi-k2.6 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k26.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "moonshot/k26",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.112903,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "stepfun/3.5-flash",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.120968,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "stepfun/3.7-flash",
      "problems": 31,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.058036,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Claude-Opus-4.6 (High)', model_config 'anthropic/opus_46' (claude-opus-4-6, extended thinking budget 120000, effort high, per configs/models/anthropic/opus_46.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "anthropic/opus_46",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.058036,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "anthropic/opus_47",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.357143,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "anthropic/opus_48_max",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.138393,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "gemini/gemini-31-pro",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.160714,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "gemini/gemini-35-flash",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.120536,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'GLM 5', model_config 'glm/glm-5' (glm-5 (Z.ai API), temperature 1.0, per configs/models/glm/glm-5.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "glm/glm-5",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.064732,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "glm/glm-51",
      "problems": 56,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.120536,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "glm/glm-52",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.366071,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "openai/gpt-54",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.736607,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "openai/gpt-55",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k2-6",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.15625,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Kimi K2.6 (Think)', model_config 'moonshot/k26' (moonshotai/kimi-k2.6 via OpenRouter pinned to Moonshot AI, reasoning high, per configs/models/moonshot/k26.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "moonshot/k26",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.071429,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "stepfun/3.5-flash",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.107143,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "stepfun/3.7-flash",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.540984,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "anthropic/fable_5_max",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.803279,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Claude-Fable-5.1 (max)', model_config 'anthropic/fable_51_max' (claude-fable-5-1, adaptive thinking, effort max, per configs/models/anthropic/fable_51_max.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "anthropic/fable_51_max",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.040984,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Claude-Opus-4.7 (xhigh)', model_config 'anthropic/opus_47' (claude-opus-4-7, adaptive thinking, effort xhigh, per configs/models/anthropic/opus_47.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "anthropic/opus_47",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.344262,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "anthropic/opus_48_max",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.258197,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "gemini/gemini-31-pro",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.213115,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "gemini/gemini-35-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.159836,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Gemini 3.6 Flash', model_config 'gemini/gemini-36-flash' (gemini-3.6-flash (Google AI Studio via INSAIT proxy), high, per configs/models/gemini/gemini-36-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "gemini/gemini-36-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.192623,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Gemini 3.7 Flash', model_config 'gemini/gemini-37-flash' (gemini-3.7-flash, reasoning_effort high, per configs/models/gemini/gemini-37-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "gemini/gemini-37-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.336066,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (gemini-3.8-flash, reasoning_effort high, per configs/models/gemini/gemini-38-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "gemini/gemini-38-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.081967,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "glm/glm-51",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.184426,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "glm/glm-52",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.721311,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "openai/gpt-55",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.97541,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (gpt-6-astra, reasoning_effort max, per configs/models/openai/gpt-6-astra.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "openai/gpt-6-astra",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.594262,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (moonshotai/kimi-k3 via OpenRouter pinned to Moonshot AI, reasoning max, per configs/models/moonshot/k3.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "moonshot/k3",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.454918,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (qwen/qwen3.8-max via OpenRouter pinned to Alibaba (HF id Qwen/Qwen3.8-2.4T-A95B), reasoning max, per configs/models/qwen/qwen3.8_max.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "qwen/qwen3.8_max",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.139344,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "stepfun/3.5-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.188525,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "stepfun/3.7-flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.445,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "anthropic/fable_5_max",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.23,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "anthropic/opus_48_max",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.17,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "gemini/gemini-31-pro",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.11,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "gemini/gemini-35-flash",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.1,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "glm/glm-51",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.105,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "glm/glm-52",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.5,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "openai/gpt-55",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.085,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "stepfun/3.7-flash",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.478395,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max' (claude-fable-5, adaptive thinking, effort max, per configs/models/anthropic/fable_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "anthropic/fable_5_max",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.416667,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max' (claude-opus-4-8, adaptive thinking, effort max, per configs/models/anthropic/opus_48_max.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "anthropic/opus_48_max",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-5",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.907407,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Claude-Opus-5 (max)', model_config 'anthropic/opus_5_max' (claude-opus-5, adaptive thinking, effort max, per configs/models/anthropic/opus_5_max.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "anthropic/opus_5_max",
      "problems": 54,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.175926,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "gemini/gemini-31-pro",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.123457,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Gemini 3.5 Flash', model_config 'gemini/gemini-35-flash' (gemini-3.5-flash, reasoning_effort high, per configs/models/gemini/gemini-35-flash.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "gemini/gemini-35-flash",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.12037,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Gemini 3.6 Flash', model_config 'gemini/gemini-36-flash' (gemini-3.6-flash (Google AI Studio via INSAIT proxy), high, per configs/models/gemini/gemini-36-flash.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "gemini/gemini-36-flash",
      "problems": 54,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.141975,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'GLM 5.2', model_config 'glm/glm-52' (glm-5.2 (BigModel API), thinking enabled, reasoning_effort max, per configs/models/glm/glm-52.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "glm/glm-52",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.694444,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "openai/gpt-55",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.67284,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'GPT-5.6-Sol (max)', model_config 'openai/gpt-56-sol' (gpt-5.6-sol, reasoning effort max, per configs/models/openai/gpt-56-sol.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "openai/gpt-56-sol",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "grok-4-5",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.29321,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Grok 4.5', model_config 'xai/grok-45' (grok-4.5 (xAI API), reasoning_effort high, per configs/models/xai/grok-45.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "xai/grok-45",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.518519,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (moonshotai/kimi-k3 via OpenRouter pinned to Moonshot AI, reasoning max, per configs/models/moonshot/k3.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "moonshot/k3",
      "problems": 54,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "muse-spark-1-1",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.506173,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Muse Spark 1.1', model_config 'meta/muse_spark_11' (meta/muse-spark-1.1 via OpenRouter, per configs/models/meta/muse_spark_11.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "meta/muse_spark_11",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.027778,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Qwen3.5-2B', model_config 'qwen/qwen3.5_2b' (Qwen3.5-2B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.5_2b.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "qwen/qwen3.5_2b",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.033951,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Qwen3.6-35B', model_config 'qwen/qwen3.6_35b' (Qwen/Qwen3.6-35B-A3B self-hosted vLLM, thinking enabled, per configs/models/qwen/qwen3.6_35b.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "qwen/qwen3.6_35b",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-brokenarxiv-2026-06",
    "score": 0.157407,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626_outputs/blob/c4fac0ea2939a252ea72e9712fa49c639becb252/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-08-25",
    "note": "MathArena BrokenArXiv June 2026 outputs dataset (HF revision c4fac0ea2939a252ea72e9712fa49c639becb252); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (step-3.7-flash (StepFun API), reasoning_effort high, per configs/models/stepfun/3.7-flash.yaml at eth-sri/matharena b89f2f0ad64c). 54/54 problems, 3 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0626_outputs",
      "revision": "c4fac0ea2939a252ea72e9712fa49c639becb252",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2d331b9064f4bee5894b0ae4413048eb21724b2d68bf1d459b46301544b6c1a1",
      "model_config": "stepfun/3.7-flash",
      "problems": 54,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0.146341,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'Claude-Opus-4.7 (high)', model_config 'anthropic/opus_47_high' (claude-opus-4-7, adaptive thinking, effort high, per configs/models/anthropic/opus_47_high.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "anthropic/opus_47_high",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 1
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0.146341,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (gemini-3.1-pro-preview, thinking_level high, per configs/models/gemini/gemini-31-pro.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1-4 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "gemini/gemini-31-pro",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "glm-5-1",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0.02439,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'GLM 5.1', model_config 'glm/glm-51' (glm-5.1 (Z.ai API), temperature 1.0, per configs/models/glm/glm-51.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "glm/glm-51",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 1
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-4",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0.170732,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'GPT-5.4 (xhigh)', model_config 'openai/gpt-54' (gpt-5.4, reasoning effort xhigh, per configs/models/openai/gpt-54.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "openai/gpt-54",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 1
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0.170732,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'GPT-5.5 (xhigh)', model_config 'openai/gpt-55' (gpt-5.5, reasoning effort xhigh, per configs/models/openai/gpt-55.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "openai/gpt-55",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 1
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "step-3-5-flash",
    "bench": "matharena-arxivlean-2026-03",
    "score": 0,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0326_outputs/blob/47c34bc76c06596e0010c5c15a93c23ca9f082ed/data/train-00000-of-00001.parquet",
    "source_id": "matharena-arxiv-outputs-bysa",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean March 2026 outputs dataset (HF revision 47c34bc76c06596e0010c5c15a93c23ca9f082ed); published model_name 'Step 3.5 Flash', model_config 'stepfun/3.5-flash' (step-3.5-flash (StepFun API), per configs/models/stepfun/3.5-flash.yaml at eth-sri/matharena b89f2f0ad64c). 41/41 problems, 1 attempts per problem; score = mean Lean-verified proof success, averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used with MathArena's Lean tool loop (verify_lean, loogle, etc.; max 200 tool calls). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivlean-0326_outputs",
      "revision": "47c34bc76c06596e0010c5c15a93c23ca9f082ed",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "8b860f55a6c9e26961e51eac66cf79ec756ee2462b5dae89a4350c98579b750a",
      "model_config": "stepfun/3.5-flash",
      "problems": 41,
      "attempts_min": 1,
      "attempts_max": 1
    },
    "data_license": "CC-BY-SA-4.0"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "eqbench3-elo",
    "score": 1215.93,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'Qwen/Qwen3-235B-A22B'; raw TrueSkill rating 'elo' 1215.93 (sigma 4.26, 95% CI 1207.58-1224.27); normalised display value elo_norm 1281.78 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "qwen3-32b",
    "bench": "eqbench3-elo",
    "score": 1141.48,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'Qwen/Qwen3-32B'; raw TrueSkill rating 'elo' 1141.48 (sigma 4.17, 95% CI 1133.31-1149.64); normalised display value elo_norm 1057.46 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "qwen3-8b",
    "bench": "eqbench3-elo",
    "score": 1083.33,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'Qwen/Qwen3-8B'; raw TrueSkill rating 'elo' 1083.33 (sigma 4.71, 95% CI 1074.11-1092.55); normalised display value elo_norm 882.24 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "eqbench3-elo",
    "score": 1308.43,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-opus-4-5-20251101'; raw TrueSkill rating 'elo' 1308.43 (sigma 4.91, 95% CI 1298.82-1318.05); normalised display value elo_norm 1560.52 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "eqbench3-elo",
    "score": 1383.38,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-opus-4-6'; raw TrueSkill rating 'elo' 1383.38 (sigma 5.11, 95% CI 1373.35-1393.4); normalised display value elo_norm 1786.32 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "eqbench3-elo",
    "score": 1293.07,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-sonnet-4.5'; raw TrueSkill rating 'elo' 1293.07 (sigma 4.77, 95% CI 1283.71-1302.42); normalised display value elo_norm 1514.21 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "eqbench3-elo",
    "score": 1172.16,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'deepseek/deepseek-chat-v3-0324'; raw TrueSkill rating 'elo' 1172.16 (sigma 4.48, 95% CI 1163.37-1180.95); normalised display value elo_norm 1149.9 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "deepseek-r1",
    "bench": "eqbench3-elo",
    "score": 1224.69,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'deepseek/deepseek-r1'; raw TrueSkill rating 'elo' 1224.69 (sigma 4.77, 95% CI 1215.35-1234.04); normalised display value elo_norm 1308.19 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "eqbench3-elo",
    "score": 1307.1,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gemini-3-pro-preview'; raw TrueSkill rating 'elo' 1307.1 (sigma 4.52, 95% CI 1298.24-1315.97); normalised display value elo_norm 1556.51 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemma-3-27b",
    "bench": "eqbench3-elo",
    "score": 1159.44,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemma-3-27b-it'; raw TrueSkill rating 'elo' 1159.44 (sigma 4.06, 95% CI 1151.48-1167.4); normalised display value elo_norm 1111.58 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemma-3-4b",
    "bench": "eqbench3-elo",
    "score": 1118.18,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemma-3-4b-it'; raw TrueSkill rating 'elo' 1118.18 (sigma 3.29, 95% CI 1111.74-1124.62); normalised display value elo_norm 987.25 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-5-1",
    "bench": "eqbench3-elo",
    "score": 1314.42,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gpt-5.1-2025-11-13'; raw TrueSkill rating 'elo' 1314.42 (sigma 4.48, 95% CI 1305.64-1323.2); normalised display value elo_norm 1578.55 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-5-2",
    "bench": "eqbench3-elo",
    "score": 1314.25,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gpt-5.2'; raw TrueSkill rating 'elo' 1314.25 (sigma 4.49, 95% CI 1305.44-1323.05); normalised display value elo_norm 1578.03 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "eqbench3-elo",
    "score": 1058.37,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'meta-llama/Llama-3.1-405B-Instruct'; raw TrueSkill rating 'elo' 1058.37 (sigma 5.52, 95% CI 1047.55-1069.19); normalised display value elo_norm 807.03 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "eqbench3-elo",
    "score": 856.91,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'meta-llama/llama-3.2-1b-instruct'; raw TrueSkill rating 'elo' 856.91 (sigma 11.01, 95% CI 835.33-878.49); normalised display value elo_norm 200.0 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "llama-4-maverick",
    "bench": "eqbench3-elo",
    "score": 1062.05,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'meta-llama/llama-4-maverick'; raw TrueSkill rating 'elo' 1062.05 (sigma 4.33, 95% CI 1053.58-1070.53); normalised display value elo_norm 818.14 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "llama-4-scout",
    "bench": "eqbench3-elo",
    "score": 993.64,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'meta-llama/llama-4-scout'; raw TrueSkill rating 'elo' 993.64 (sigma 4.9, 95% CI 984.03-1003.25); normalised display value elo_norm 612.0 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "mistral-large-2",
    "bench": "eqbench3-elo",
    "score": 1108.92,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'mistralai/Mistral-Large-Instruct-2411'; raw TrueSkill rating 'elo' 1108.92 (sigma 6.11, 95% CI 1096.95-1120.9); normalised display value elo_norm 959.36 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "eqbench3-elo",
    "score": 1162.28,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'mistralai/mistral-small-3.2-24b-instruct'; raw TrueSkill rating 'elo' 1162.28 (sigma 5.91, 95% CI 1150.7-1173.86); normalised display value elo_norm 1120.14 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "kimi-k2",
    "bench": "eqbench3-elo",
    "score": 1311,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'moonshotai/Kimi-K2-Instruct'; raw TrueSkill rating 'elo' 1311.0 (sigma 4.32, 95% CI 1302.53-1319.48); normalised display value elo_norm 1568.25 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "eqbench3-elo",
    "score": 1139.82,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'nvidia/llama-3.1-nemotron-ultra-253b-v1:free'; raw TrueSkill rating 'elo' 1139.82 (sigma 3.91, 95% CI 1132.16-1147.48); normalised display value elo_norm 1052.46 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "o3",
    "bench": "eqbench3-elo",
    "score": 1288.35,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'o3'; raw TrueSkill rating 'elo' 1288.35 (sigma 4.41, 95% CI 1279.71-1296.99); normalised display value elo_norm 1500.0 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "o4-mini",
    "bench": "eqbench3-elo",
    "score": 1224.11,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'o4-mini'; raw TrueSkill rating 'elo' 1224.11 (sigma 4.23, 95% CI 1215.82-1232.4); normalised display value elo_norm 1306.44 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-4-1",
    "bench": "eqbench3-elo",
    "score": 1206.01,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-4.1'; raw TrueSkill rating 'elo' 1206.01 (sigma 4.51, 95% CI 1197.17-1214.85); normalised display value elo_norm 1251.89 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "eqbench3-elo",
    "score": 1170.96,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-4.1-mini'; raw TrueSkill rating 'elo' 1170.96 (sigma 4.42, 95% CI 1162.29-1179.63); normalised display value elo_norm 1146.28 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "eqbench3-elo",
    "score": 1167.51,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-oss-120b'; raw TrueSkill rating 'elo' 1167.51 (sigma 5.39, 95% CI 1156.94-1178.07); normalised display value elo_norm 1135.88 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "eqbench3-elo",
    "score": 1111.46,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-oss-20b'; raw TrueSkill rating 'elo' 1111.46 (sigma 5.53, 95% CI 1100.63-1122.29); normalised display value elo_norm 967.01 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "eqbench3-elo",
    "score": 1076.05,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'qwen/qwen-2.5-72b-instruct'; raw TrueSkill rating 'elo' 1076.05 (sigma 3.77, 95% CI 1068.66-1083.43); normalised display value elo_norm 860.29 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "glm-4-5",
    "bench": "eqbench3-elo",
    "score": 1229.22,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'zai-org/GLM-4.5'; raw TrueSkill rating 'elo' 1229.22 (sigma 7.77, 95% CI 1213.99-1244.45); normalised display value elo_norm 1321.82 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "glm-4-7",
    "bench": "eqbench3-elo",
    "score": 1271.34,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'zai-org/GLM-4.7'; raw TrueSkill rating 'elo' 1271.34 (sigma 4.62, 95% CI 1262.29-1280.39); normalised display value elo_norm 1448.75 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "eqbench3-elo",
    "score": 1151.26,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'zai-org/GLM-4.7-Flash'; raw TrueSkill rating 'elo' 1151.26 (sigma 4.89, 95% CI 1141.68-1160.85); normalised display value elo_norm 1086.94 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "eqbench3-elo",
    "score": 1384.22,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-sonnet-4-6'; raw TrueSkill rating 'elo' 1384.22 (sigma 6.2, 95% CI 1372.08-1396.37); normalised display value elo_norm 1788.88 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "eqbench3-elo",
    "score": 1262.92,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'qwen/qwen3.5-397b-a17b'; raw TrueSkill rating 'elo' 1262.92 (sigma 5.39, 95% CI 1252.36-1273.48); normalised display value elo_norm 1423.37 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "glm-5",
    "bench": "eqbench3-elo",
    "score": 1307.3,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'z-ai/glm-5'; raw TrueSkill rating 'elo' 1307.3 (sigma 5.33, 95% CI 1296.86-1317.75); normalised display value elo_norm 1557.11 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "kimi-k2-5",
    "bench": "eqbench3-elo",
    "score": 1303.47,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'moonshotai/kimi-k2.5'; raw TrueSkill rating 'elo' 1303.47 (sigma 4.6, 95% CI 1294.46-1312.48); normalised display value elo_norm 1545.55 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "eqbench3-elo",
    "score": 1301.76,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemini-3.1-pro-preview'; raw TrueSkill rating 'elo' 1301.76 (sigma 4.16, 95% CI 1293.61-1309.91); normalised display value elo_norm 1540.41 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-5-4",
    "bench": "eqbench3-elo",
    "score": 1327.99,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-5.4'; raw TrueSkill rating 'elo' 1327.99 (sigma 5.59, 95% CI 1317.03-1338.94); normalised display value elo_norm 1619.42 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemma-4-31b",
    "bench": "eqbench3-elo",
    "score": 1272.94,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemma-4-31b-it'; raw TrueSkill rating 'elo' 1272.94 (sigma 4.55, 95% CI 1264.03-1281.85); normalised display value elo_norm 1453.57 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "eqbench3-elo",
    "score": 1272.28,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemma-4-26b-a4b-it'; raw TrueSkill rating 'elo' 1272.28 (sigma 5.94, 95% CI 1260.63-1283.92); normalised display value elo_norm 1451.56 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "glm-5-1",
    "bench": "eqbench3-elo",
    "score": 1309.62,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'z-ai/glm-5.1'; raw TrueSkill rating 'elo' 1309.62 (sigma 5.39, 95% CI 1299.05-1320.19); normalised display value elo_norm 1564.08 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "eqbench3-elo",
    "score": 1460.91,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-opus-4-7'; raw TrueSkill rating 'elo' 1460.91 (sigma 9.93, 95% CI 1441.45-1480.37); normalised display value elo_norm 2019.95 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-5-5",
    "bench": "eqbench3-elo",
    "score": 1327.87,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'openai/gpt-5.5'; raw TrueSkill rating 'elo' 1327.87 (sigma 5.62, 95% CI 1316.85-1338.89); normalised display value elo_norm 1619.09 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "kimi-k2-6",
    "bench": "eqbench3-elo",
    "score": 1319.24,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "source_id": "eqbench3-canonical-results",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'moonshotai/kimi-k2.6'; raw TrueSkill rating 'elo' 1319.24 (sigma 5.42, 95% CI 1308.61-1329.87); normalised display value elo_norm 1593.08 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data."
  },
  {
    "model": "gpt-4-1",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.918,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-2025-04-14'; column 'Model Avg' = 0.918; single-turn 0.94, multi-turn 0.89; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.91,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-mini-2025-04-14'; column 'Model Avg' = 0.910; single-turn 0.93, multi-turn 0.89; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.905,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'deepseek-v3-0324'; column 'Model Avg' = 0.905; single-turn 0.91, multi-turn 0.90; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "o4-mini",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.889,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'o4-mini-2025-04-16'; column 'Model Avg' = 0.889; single-turn 0.88, multi-turn 0.90; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "o3",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.861,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'o3-2025-04-16'; column 'Model Avg' = 0.861; single-turn 0.86, multi-turn 0.86; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "nova-micro",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.829,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'amazon.nova-micro-v1'; column 'Model Avg' = 0.829; single-turn 0.90, multi-turn 0.75; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.817,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'qwen2.5-72b-instruct'; column 'Model Avg' = 0.817; single-turn 0.80, multi-turn 0.84; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "mistral-large-2",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.81,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'mistral-large-2411'; column 'Model Avg' = 0.810; single-turn 0.87, multi-turn 0.75; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.774,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'Llama-3.3-70B-Instruct-Turbo'; column 'Model Avg' = 0.774; single-turn 0.86, multi-turn 0.69; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "llama-4-maverick",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.741,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'Llama-4-Maverick-17B-128E-Instruct-FP8'; column 'Model Avg' = 0.741; single-turn 0.78, multi-turn 0.70; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.678,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'Meta-Llama-3.1-8B-Instruct-Turbo'; column 'Model Avg' = 0.678; single-turn 0.71, multi-turn 0.64; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "llama-4-scout",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.629,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'Llama-4-Scout-17B-16E-Instruct'; column 'Model Avg' = 0.629; single-turn 0.69, multi-turn 0.57; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets."
  },
  {
    "model": "gpt-4-1",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.62,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-2025-04-14'; column 'Avg AC' = 0.62; Avg TSQ 0.8; per-domain AC banking 0.6, healthcare 0.62, insurance 0.66, investment 0.64, telecom 0.58; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.56,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-mini-2025-04-14'; column 'Avg AC' = 0.56; Avg TSQ 0.79; per-domain AC banking 0.56, healthcare 0.6, insurance 0.46, investment 0.5, telecom 0.64; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "kimi-k2",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.53,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'kimi-k2-instruct'; column 'Avg AC' = 0.53; Avg TSQ 0.9; per-domain AC banking 0.58, healthcare 0.49, insurance 0.58, investment 0.47, telecom 0.53; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.53,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'qwen3-235b-a22b-instruct-2507'; column 'Avg AC' = 0.53; Avg TSQ 0.85; per-domain AC banking 0.44, healthcare 0.49, insurance 0.74, investment 0.41, telecom 0.58; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.51,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'qwen2.5-72b-instruct'; column 'Avg AC' = 0.51; Avg TSQ 0.8; per-domain AC banking 0.48, healthcare 0.61, insurance 0.52, investment 0.42, telecom 0.52; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.47,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.5-flash-lite'; column 'Avg AC' = 0.47; Avg TSQ 0.84; per-domain AC banking 0.45, healthcare 0.6, insurance 0.54, investment 0.35, telecom 0.41; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "glm-4-5-air",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.44,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'glm-4.5-air'; column 'Avg AC' = 0.44; Avg TSQ 0.94; per-domain AC banking 0.49, healthcare 0.4, insurance 0.53, investment 0.46, telecom 0.33; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.43,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.5-pro'; column 'Avg AC' = 0.43; Avg TSQ 0.86; per-domain AC banking 0.45, healthcare 0.4, insurance 0.54, investment 0.31, telecom 0.44; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.38,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.5-flash'; column 'Avg AC' = 0.38; Avg TSQ 0.94; per-domain AC banking 0.48, healthcare 0.38, insurance 0.44, investment 0.22, telecom 0.36; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.34,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'qwen3-235b-a22b-thinking-2507'; column 'Avg AC' = 0.34; Avg TSQ 0.85; per-domain AC banking 0.42, healthcare 0.3, insurance 0.42, investment 0.23, telecom 0.34; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.26,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'mistral-small-2506'; column 'Avg AC' = 0.26; Avg TSQ 0.71; per-domain AC banking 0.37, healthcare 0.28, insurance 0.22, investment 0.2, telecom 0.21; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.2,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'llama-3.3-70b-instruct'; column 'Avg AC' = 0.2; Avg TSQ 0.62; per-domain AC banking 0.11, healthcare 0.29, insurance 0.29, investment 0.14, telecom 0.16; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric."
  },
  {
    "model": "o3",
    "bench": "simple-evals-mmlu",
    "score": 0.933,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'MMLU' = 93.3; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 93.3,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.834,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'GPQA' = 83.4; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-04-14.",
    "raw_value": 83.4,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-math500",
    "score": 0.981,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'MATH' = 98.1; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-04-14.",
    "raw_value": 98.1,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-humaneval",
    "score": 0.884,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'HumanEval' = 88.4; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 88.4,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-mgsm",
    "score": 0.92,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'MGSM' = 92.0; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 92,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-drop-f1",
    "score": 0.898,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'DROP (F1, 3-shot)' = 89.8; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 89.8,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "simple-evals-simpleqa",
    "score": 0.486,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-high [^10]', column 'SimpleQA' = 48.6; prompt 'n/a [^7]'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 48.6,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-mmlu",
    "score": 0.903,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'MMLU' = 90.3; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 90.3,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.813,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'GPQA' = 81.3; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-04-14.",
    "raw_value": 81.3,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-math500",
    "score": 0.982,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'MATH' = 98.2; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-04-14.",
    "raw_value": 98.2,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-humaneval",
    "score": 0.993,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'HumanEval' = 99.3; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 99.3,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-mgsm",
    "score": 0.935,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'MGSM' = 93.5; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 93.5,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-drop-f1",
    "score": 0.781,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'DROP (F1, 3-shot)' = 78.1; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 78.1,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "simple-evals-simpleqa",
    "score": 0.193,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o4-mini-high [^9] [^10]', column 'SimpleQA' = 19.3; prompt 'n/a'. Reasoning effort high (highest listed row). No tools enabled for o3/o4-mini (footnote 10). Row added 2025-04-14.",
    "raw_value": 19.3,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-mmlu",
    "score": 0.902,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'MMLU' = 90.2; prompt 'assistant [^2]'. Row added 2025-04-14.",
    "raw_value": 90.2,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.663,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'GPQA' = 66.3; prompt 'assistant [^2]'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-04-14.",
    "raw_value": 66.3,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-math500",
    "score": 0.821,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'MATH' = 82.1; prompt 'assistant [^2]'. MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-04-14.",
    "raw_value": 82.1,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-humaneval",
    "score": 0.945,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'HumanEval' = 94.5; prompt 'assistant [^2]'. Row added 2025-04-14.",
    "raw_value": 94.5,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-mgsm",
    "score": 0.869,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'MGSM' = 86.9; prompt 'assistant [^2]'. Row added 2025-04-14.",
    "raw_value": 86.9,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-drop-f1",
    "score": 0.794,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'DROP (F1, 3-shot)' = 79.4; prompt 'assistant [^2]'. Row added 2025-04-14.",
    "raw_value": 79.4,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "simple-evals-simpleqa",
    "score": 0.416,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-2025-04-14', column 'SimpleQA' = 41.6; prompt 'assistant [^2]'. Row added 2025-04-14.",
    "raw_value": 41.6,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-mmlu",
    "score": 0.875,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'MMLU' = 87.5; prompt 'assistant'. Row added 2025-04-14.",
    "raw_value": 87.5,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.65,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'GPQA' = 65.0; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-04-14.",
    "raw_value": 65,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-math500",
    "score": 0.814,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'MATH' = 81.4; prompt 'assistant'. MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-04-14.",
    "raw_value": 81.4,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-humaneval",
    "score": 0.938,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'HumanEval' = 93.8; prompt 'assistant'. Row added 2025-04-14.",
    "raw_value": 93.8,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-mgsm",
    "score": 0.882,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'MGSM' = 88.2; prompt 'assistant'. Row added 2025-04-14.",
    "raw_value": 88.2,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-drop-f1",
    "score": 0.81,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'DROP (F1, 3-shot)' = 81.0; prompt 'assistant'. Row added 2025-04-14.",
    "raw_value": 81,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "simple-evals-simpleqa",
    "score": 0.168,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "source_id": "openai-simple-evals-readme",
    "by": "lab",
    "date": "2025-04-14",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-mini-2025-04-14', column 'SimpleQA' = 16.8; prompt 'assistant'. Row added 2025-04-14.",
    "raw_value": 16.8,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.859,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'o3-2025-04-16' = 85.9% (CI --0.8/+0.9); win rate vs baseline o3-mini-2025-01-31 (50%); o3-2025-04-16 at default (medium) reasoning effort; no '-high' row published. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 85.9,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.791,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'o4-mini-2025-04-16-high' = 79.1% (CI --1.4/+1.2); win rate vs baseline o3-mini-2025-01-31 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 79.1,
    "scale_divisor": 100
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.584,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Qwen3-235B-A22B' = 58.4% (CI --1.9/+2.1); win rate vs baseline o3-mini-2025-01-31 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 58.4,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-r1",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.58,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'deepseek-r1' = 58.0% (CI --2.2/+2.0); win rate vs baseline o3-mini-2025-01-31 (50%); listed Apr/May 2025, before R1-0528. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 58,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.5,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'gpt-4.1' = 50.0% (CI --1.9/+1.7); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 50,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.469,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'gpt-4.1-mini' = 46.9% (CI --2.4/+2.1); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 46.9,
    "scale_divisor": 100
  },
  {
    "model": "qwen3-32b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.445,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Qwen3-32B' = 44.5% (CI --2.2/+2.1); win rate vs baseline o3-mini-2025-01-31 (50%); thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 44.5,
    "scale_divisor": 100
  },
  {
    "model": "llama-4-maverick",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.172,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'llama4-maverick-instruct-basic' = 17.2% (CI --1.5/+1.2); win rate vs baseline o3-mini-2025-01-31 (50%); Fireworks 'basic' deployment name of Llama 4 Maverick Instruct. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 17.2,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-27b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.15,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'gemma-3-27b-it' = 15.0% (CI --1.4/+1.0); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 15,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-72b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.101,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Qwen2.5-72B-Instruct' = 10.1% (CI --0.9/+1.3); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 10.1,
    "scale_divisor": 100
  },
  {
    "model": "o3",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.888,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'o3-2025-04-16' = 88.8% (CI --1.1/+1.0); win rate vs baseline gemini-2.0-flash-001 (50%); o3-2025-04-16 at default (medium) reasoning effort; no '-high' row published. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 88.8,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-r1",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.77,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'deepseek-r1' = 77.0% (CI --2.0/+1.4); win rate vs baseline gemini-2.0-flash-001 (50%); listed Apr/May 2025, before R1-0528. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 77,
    "scale_divisor": 100
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.735,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Qwen3-235B-A22B' = 73.5% (CI --1.8/+1.5); win rate vs baseline gemini-2.0-flash-001 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 73.5,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-27b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.699,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'gemma-3-27b-it' = 69.9% (CI --1.9/+1.7); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 69.9,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.615,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'gpt-4.1' = 61.5% (CI --1.9/+1.9); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 61.5,
    "scale_divisor": 100
  },
  {
    "model": "o4-mini",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.587,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'o4-mini-2025-04-16-high' = 58.7% (CI --1.8/+1.9); win rate vs baseline gemini-2.0-flash-001 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 58.7,
    "scale_divisor": 100
  },
  {
    "model": "qwen3-32b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.533,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Qwen3-32B' = 53.3% (CI --1.9/+1.6); win rate vs baseline gemini-2.0-flash-001 (50%); thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 53.3,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.282,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'gpt-4.1-mini' = 28.2% (CI --1.8/+1.8); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 28.2,
    "scale_divisor": 100
  },
  {
    "model": "llama-4-maverick",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.105,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'llama4-maverick-instruct-basic' = 10.5% (CI --1.1/+1.0); win rate vs baseline gemini-2.0-flash-001 (50%); Fireworks 'basic' deployment name of Llama 4 Maverick Instruct. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 10.5,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-72b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.102,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "source_id": "arena-hard-auto-readme-v2",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Qwen2.5-72B-Instruct' = 10.2% (CI --1.1/+1.1); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 10.2,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-r1",
    "bench": "scicode-main-official",
    "score": 0.046,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "source_id": "scicode-readme-leaderboard",
    "by": "independent",
    "date": "2025-02-01",
    "note": "SciCode repository README leaderboard; row 'Deepseek-R1'; Main Problem Resolve Rate 4.6% (subproblem 28.5% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; results for DeepSeek models added 2025-02-01 per README news.",
    "raw_value": 4.6,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-v3",
    "bench": "scicode-main-official",
    "score": 0.031,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "source_id": "scicode-readme-leaderboard",
    "by": "independent",
    "date": "2025-02-01",
    "note": "SciCode repository README leaderboard; row 'Deepseek-v3'; Main Problem Resolve Rate 3.1% (subproblem 23.7% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; results for DeepSeek models added 2025-02-01 per README news.",
    "raw_value": 3.1,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-1-405b",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "source_id": "scicode-readme-leaderboard",
    "by": "independent",
    "date": "2025-02-01",
    "note": "SciCode repository README leaderboard; row 'Llama-3.1-405B-Chat'; Main Problem Resolve Rate 1.5% (subproblem 19.8% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; results for DeepSeek models added 2025-02-01 per README news.",
    "raw_value": 1.5,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-1-70b",
    "bench": "scicode-main-official",
    "score": 0,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "source_id": "scicode-readme-leaderboard",
    "by": "independent",
    "date": "2025-02-01",
    "note": "SciCode repository README leaderboard; row 'Llama-3.1-70B-Chat'; Main Problem Resolve Rate 0.0% (subproblem 17.0% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; results for DeepSeek models added 2025-02-01 per README news.",
    "raw_value": 0,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-r1",
    "bench": "judgemark-v2",
    "score": 0.7697,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "source_id": "judgemark-v2-results",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'deepseek-ai/deepseek-r1'; judgemark_score 76.97 (calibrated, 0-100), raw 71.77, Kendall tau 0.863. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 76.97,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "judgemark-v2",
    "score": 0.5967,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "source_id": "judgemark-v2-results",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'deepseek-ai/deepseek-r1-distill-llama-70b'; judgemark_score 59.67 (calibrated, 0-100), raw 56.86, Kendall tau 0.764. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 59.67,
    "scale_divisor": 100
  },
  {
    "model": "mistral-large-2",
    "bench": "judgemark-v2",
    "score": 0.6563,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "source_id": "judgemark-v2-results",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'mistralai/mistral-large-instruct-2411'; judgemark_score 65.63 (calibrated, 0-100), raw 63.25, Kendall tau 0.794. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 65.63,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-1-8b",
    "bench": "judgemark-v2",
    "score": 0.3572,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "source_id": "judgemark-v2-results",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'meta-llama/llama-3.1-8b-instruct'; judgemark_score 35.72 (calibrated, 0-100), raw 35.67, Kendall tau 0.46. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 35.72,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-1-405b",
    "bench": "judgemark-v2",
    "score": 0.6009,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "source_id": "judgemark-v2-results",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'meta-llama/llama-3.1-405b-instruct'; judgemark_score 60.09 (calibrated, 0-100), raw 58.76, Kendall tau 0.791. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 60.09,
    "scale_divisor": 100
  },
  {
    "model": "claude-opus-4-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.402778,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-5-20251101-thinking-64k/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-opus-4-5-20251101-thinking-64k' (thinking budget 64k; API model in attempt metadata: claude-opus-4-5-20251101). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 48.333333333333336 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.790278,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-6-thinking-120K-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-opus-4-6-thinking-120K-high' (thinking budget 120k, effort high; API model in attempt metadata: claude-opus-4-6). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 94.83333333333333 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.769444,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-8-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-opus-4-8-high' (adaptive thinking, effort high; API model in attempt metadata: claude-opus-4-8). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 92.33333333333333 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.038889,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/deepseek-v3.2",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'deepseek-v3.2' (Fireworks deepseek-v3p2, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/deepseek-v3p2). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 4.667 / 120); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.340278,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-flash-preview-thinking-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gemini-3-flash-preview-thinking-high' (thinking level high). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 40.833333333333336 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "glm-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.054167,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/glm-5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'glm-5' (Fireworks glm-5, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/glm-5). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 6.5 / 120); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "gpt-5-1",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.183333,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-1-2025-11-13-thinking-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-1-2025-11-13-thinking-high' (reasoning effort high; API model in attempt metadata: gpt-5.1-2025-11-13). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 22.0 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "gpt-5-2",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.276389,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-2-2025-12-11-thinking-medium/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-2-2025-12-11-thinking-medium' (reasoning effort medium). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 33.16666666666667 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "kimi-k2-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.120833,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/kimi-k2.5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'kimi-k2.5' (Fireworks kimi-k2p5, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/kimi-k2p5). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 14.5 / 120); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "minimax-m2-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.054167,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/minimax-m2.5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'minimax-m2.5' (MiniMax API minimax/minimax-m2.5; API model in attempt metadata: minimax/minimax-m2.5). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 6.5 / 120); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.86625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-5-20251101-thinking-32k/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-opus-4-5-20251101-thinking-32k' (thinking budget 32k; API model in attempt metadata: claude-opus-4-5-20251101). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 346.5 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.9675,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-6-thinking-120K-max/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-opus-4-6-thinking-120K-max' (thinking budget 120k, effort max; API model in attempt metadata: claude-opus-4-6). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 387.0 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.955,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-8-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-opus-4-8-high' (adaptive thinking, effort high; API model in attempt metadata: claude-opus-4-8). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 382.0 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.61625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/deepseek-v3.2",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'deepseek-v3.2' (Fireworks deepseek-v3p2, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/deepseek-v3p2). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 246.5 / 400); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "glm-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.58625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/glm-5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'glm-5' (Fireworks glm-5, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/glm-5). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 234.5 / 400); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "gpt-5-1",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.77125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-1-2025-11-13-thinking-high/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-1-2025-11-13-thinking-high' (reasoning effort high; API model in attempt metadata: gpt-5.1-2025-11-13). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 308.5 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "gpt-5-2",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.95,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-2-2025-12-11-thinking-xhigh/results.json",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-2-2025-12-11-thinking-xhigh' (reasoning effort xhigh; API model in attempt metadata: gpt-5.2-2025-12-11). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 380.0 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "kimi-k2-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.73125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/kimi-k2.5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'kimi-k2.5' (Fireworks kimi-k2p5, reasoning setting not recorded; API model in attempt metadata: accounts/fireworks/models/kimi-k2p5). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 292.5 / 400); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "minimax-m2-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.59125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/minimax-m2.5",
    "source_id": "arcprize-model-baseline-outputs",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'minimax-m2.5' (MiniMax API minimax/minimax-m2.5; API model in attempt metadata: minimax/minimax-m2.5). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 236.5 / 400); method reproduces the published results.json exactly for every folder that has one. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set)."
  },
  {
    "model": "deepseek-r1",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.704545,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Vector Institute Evaluation Leaderboard entry 'DeepSeek-R1', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-11T11-37-45-05-00_gpqa-diamond_MwnVeLwyuiEAALr3M5q3dn.eval). DeepSeek API model announced 2025-01-20 (config model_sha links api-docs news250120); runs Jan-Feb 2025, i.e. the original R1, not R1-0528. Reasoning settings not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "deepseek-r1",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.838265,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2025-02-12",
    "note": "Vector Institute Evaluation Leaderboard entry 'DeepSeek-R1', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-12T11-02-35-05-00_mmlu-pro_BhD89DYN9KM3k4weSDfaQK.eval). DeepSeek API model announced 2025-01-20 (config model_sha links api-docs news250120); runs Jan-Feb 2025, i.e. the original R1, not R1-0528. Reasoning settings not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "deepseek-r1",
    "bench": "vector-inspect-math",
    "score": 0.9272,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Vector Institute Evaluation Leaderboard entry 'DeepSeek-R1', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-11T11-38-10-05-00_math_ZYFSqsWsmP5kLRLHEMWULU.eval). DeepSeek API model announced 2025-01-20 (config model_sha links api-docs news250120); runs Jan-Feb 2025, i.e. the original R1, not R1-0528. Reasoning settings not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "deepseek-r1",
    "bench": "vector-inspect-humaneval",
    "score": 0.95679,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2025-02-03",
    "note": "Vector Institute Evaluation Leaderboard entry 'DeepSeek-R1', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2025-02-03T11-45-22-05-00_humaneval_hnkHWYqrb5HxiBt2CWzCnq.eval). DeepSeek API model announced 2025-01-20 (config model_sha links api-docs news250120); runs Jan-Feb 2025, i.e. the original R1, not R1-0528. Reasoning settings not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.431818,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2024-10-29",
    "note": "Vector Institute Evaluation Leaderboard entry 'Meta-Llama-3.1-70B-Instruct', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-29T23-41-39-04-00_gpqa-diamond_TdLdYmVM6GCVMAECcXkuhj.eval). meta-llama/Llama-3.1-70B-Instruct, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.668883,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'Meta-Llama-3.1-70B-Instruct', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T06-11-16-04-00_mmlu-pro_oQiEBJdeKtEEt4cm9KL7uy.eval). meta-llama/Llama-3.1-70B-Instruct, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "vector-inspect-math",
    "score": 0.6004,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'Meta-Llama-3.1-70B-Instruct', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T02-34-50-04-00_math_2xiNcrGih26uzJdG4q88bM.eval). meta-llama/Llama-3.1-70B-Instruct, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "vector-inspect-humaneval",
    "score": 0.786585,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'Meta-Llama-3.1-70B-Instruct', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2024-10-30T02-28-25-04-00_humaneval_KcJV2rHuHJ2JLxijihEkcW.eval). meta-llama/Llama-3.1-70B-Instruct, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "qwen3-coder-next",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.7563,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'Qwen3-Coder-Next', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2026-07-16T17-31-51-00-00_gpqa-diamond_UDZSZnaVwgaHozBsfMFdfa.eval). Qwen3-Coder-Next, dtype torch.bfloat16 per config (model_sha blank). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "qwen3-coder-next",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.8246,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'Qwen3-Coder-Next', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2026-07-16T19-34-49-00-00_mmlu-pro_eQivCTM2rEHS5Nfh6Zm7Dn.eval). Qwen3-Coder-Next, dtype torch.bfloat16 per config (model_sha blank). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "qwen3-coder-next",
    "bench": "vector-inspect-math",
    "score": 0.9782,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'Qwen3-Coder-Next', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2026-07-16T17-08-24-00-00_math_5s3th8ZQLsyHg3iisxF4Bd.eval). Qwen3-Coder-Next, dtype torch.bfloat16 per config (model_sha blank). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "qwen3-coder-next",
    "bench": "vector-inspect-humaneval",
    "score": 0.9146,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "source_id": "vector-inspect-evals-leaderboard",
    "by": "independent",
    "date": "2026-07-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'Qwen3-Coder-Next', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2026-07-16T17-08-24-00-00_humaneval_k8d6s3bv4Tzt4kwYjNw3tm.eval). Qwen3-Coder-Next, dtype torch.bfloat16 per config (model_sha blank). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8."
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1426,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Qwen-1.5B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B); 'Average Performance' 14.26 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2527,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Qwen-7B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B); 'Average Performance' 25.27 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2848,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Llama-8B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B); 'Average Performance' 28.48 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3428,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Qwen-14B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B); 'Average Performance' 34.28 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3979,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Llama-70B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B); 'Average Performance' 39.79 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4425,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1/tree/main); 'Average Performance' 44.25 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3975,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-Distill-Qwen-32B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B); 'Average Performance' 39.75 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3694,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'DeepSeek-R1-0528-Qwen3-8B' (model link https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B); 'Average Performance' 36.94 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-3-1b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1573,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-3-1b-it' (model link https://huggingface.co/google/gemma-3-1b-it); 'Average Performance' 15.73 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-3-4b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2856,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-3-4b-it' (model link https://huggingface.co/google/gemma-3-4b-it); 'Average Performance' 28.56 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-3-12b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3732,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-3-12b-it' (model link https://huggingface.co/google/gemma-3-12b-it); 'Average Performance' 37.32 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-3-27b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.399,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-3-27b-it' (model link https://huggingface.co/google/gemma-3-27b-it); 'Average Performance' 39.9 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-4-e2b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3274,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-04-08",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-4-E2B-it' (model link https://huggingface.co/google/gemma-4-E2B-it); 'Average Performance' 32.74 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3919,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-04-08",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-4-E4B-it' (model link https://huggingface.co/google/gemma-4-E4B-it); 'Average Performance' 39.19 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4451,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-04-08",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-4-26B-A4B-it' (model link https://huggingface.co/google/gemma-4-26B-A4B-it); 'Average Performance' 44.51 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-4-31b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4674,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-04-08",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-4-31B-it' (model link https://huggingface.co/google/gemma-4-31B-it); 'Average Performance' 46.74 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2898,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.1-8B-Instruct' (model link https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct); 'Average Performance' 28.98 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3909,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.1-70B-Instruct' (model link https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct); 'Average Performance' 39.09 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1272,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.2-1B-Instruct' (model link https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct); 'Average Performance' 12.72 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-2-3b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.229,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.2-3B-Instruct' (model link https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct); 'Average Performance' 22.9 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3986,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.3-70B-Instruct' (model link https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct); 'Average Performance' 39.86 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-4-scout",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3512,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-4-Scout-17B-16E-Instruct' (model link https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct); 'Average Performance' 35.12 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "mistral-large-2",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4228,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Mistral-Large-Instruct-2411' (model link https://huggingface.co/mistralai/Mistral-Large-Instruct-2411); 'Average Performance' 42.28 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "phi-4",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3613,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Phi-4' (model link https://huggingface.co/microsoft/phi-4); 'Average Performance' 36.13 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "phi-4-mini",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2454,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Phi-4-mini-instruct' (model link https://huggingface.co/microsoft/Phi-4-mini-instruct); 'Average Performance' 24.54 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2131,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Phi-4-mini-reasoning' (model link https://huggingface.co/microsoft/Phi-4-mini-reasoning); 'Average Performance' 21.31 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen2-5-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3132,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen2.5-7B-Instruct' (model link https://huggingface.co/Qwen/Qwen2.5-7B-Instruct); 'Average Performance' 31.32 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen2-5-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3989,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen2.5-32B-Instruct' (model link https://huggingface.co/Qwen/Qwen2.5-32B-Instruct); 'Average Performance' 39.89 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4162,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen2.5-72B-Instruct' (model link https://huggingface.co/Qwen/Qwen2.5-72B-Instruct); 'Average Performance' 41.62 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-0-6b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2038,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-0.6B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-0.6B); 'Average Performance' 20.38 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2628,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-1.7B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-1.7B); 'Average Performance' 26.28 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3998,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-8B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-8B); 'Average Performance' 39.98 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-14b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4017,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-14B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-14B); 'Average Performance' 40.17 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4104,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-32B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-32B); 'Average Performance' 41.04 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4163,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-235B-A22B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-235B-A22B); 'Average Performance' 41.63 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3406,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-4B-Instruct-2507' (model link https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507); 'Average Performance' 34.06 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4004,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-4B-Thinking-2507' (model link https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507); 'Average Performance' 40.04 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3687,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-30B-A3B-Instruct-2507' (model link https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507); 'Average Performance' 36.87 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4182,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-30B-A3B-Thinking-2507' (model link https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507); 'Average Performance' 41.82 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3981,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-11-01",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-Next-80B-A3B-Instruct' (model link https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct); 'Average Performance' 39.81 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4386,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-11-01",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-Next-80B-A3B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking); 'Average Performance' 43.86 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-8-27b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4661,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3.8-27B-Thinking' (model link https://huggingface.co/Qwen/Qwen3.8-27B); 'Average Performance' 46.61 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4695,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2026-09-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3.8-Flash-Next-Thinking' (model link https://huggingface.co/Qwen/Qwen3.8-Flash-Next); 'Average Performance' 46.95 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2905,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'gpt-oss-20b' (model link https://huggingface.co/openai/gpt-oss-20b); 'Average Performance' 29.05 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Reasoning effort not published. Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3724,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'gpt-oss-120b' (model link https://huggingface.co/openai/gpt-oss-120b); 'Average Performance' 37.24 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Reasoning effort not published. Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "robloxqa-v2-mc",
    "score": 0.850333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2551/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Qwen 3 Coder 30B-A3B' (https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct); no reasoning setting listed; run files qwen-qwen3-coder-30b-a3b-instruct_mc.json5 / qwen-qwen3-coder-30b-a3b-instruct.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.384615,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 39+34+7=80/208. Roblox LLM Leaderboard entry 'Qwen 3 Coder 30B-A3B' (https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct); no reasoning setting listed; run files qwen-qwen3-coder-30b-a3b-instruct_mc.json5 / qwen-qwen3-coder-30b-a3b-instruct.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-5-9b",
    "bench": "robloxqa-v2-mc",
    "score": 0.745,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2235/3000 correct (noAnswer=492). Roblox LLM Leaderboard entry 'Qwen 3.5 9B' (https://huggingface.co/Qwen/Qwen3.5-9B); no reasoning setting listed; run files qwen-qwen3.5-9b_mc.json5 / qwen-qwen3.5-9b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-5-9b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.192308,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 22+16+2=40/208. Roblox LLM Leaderboard entry 'Qwen 3.5 9B' (https://huggingface.co/Qwen/Qwen3.5-9B); no reasoning setting listed; run files qwen-qwen3.5-9b_mc.json5 / qwen-qwen3.5-9b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-6-27b",
    "bench": "robloxqa-v2-mc",
    "score": 0.899,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2697/3000 correct (noAnswer=46). Roblox LLM Leaderboard entry 'Qwen 3.6 27B' (https://huggingface.co/Qwen/Qwen3.6-27B); no reasoning setting listed; run files qwen-qwen3.6-27b_mc.json5 / qwen-qwen3.6-27b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-6-27b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.519231,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 41+50+17=108/208. Roblox LLM Leaderboard entry 'Qwen 3.6 27B' (https://huggingface.co/Qwen/Qwen3.6-27B); no reasoning setting listed; run files qwen-qwen3.6-27b_mc.json5 / qwen-qwen3.6-27b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "robloxqa-v2-mc",
    "score": 0.897667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2693/3000 correct (noAnswer=1). Roblox LLM Leaderboard entry 'Qwen 3.6 35B-A3B' (https://huggingface.co/Qwen/Qwen3.6-35B-A3B); no reasoning setting listed; run files qwen3.6-35b-a3b-mtp_mc.json5 / qwen-qwen3.6-35b-a3b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.5,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 42+50+12=104/208. Roblox LLM Leaderboard entry 'Qwen 3.6 35B-A3B' (https://huggingface.co/Qwen/Qwen3.6-35B-A3B); no reasoning setting listed; run files qwen3.6-35b-a3b-mtp_mc.json5 / qwen-qwen3.6-35b-a3b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "robloxqa-v2-mc",
    "score": 0.953667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2861/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Qwen 3.8 2.4T-A95B' (https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B); no reasoning setting listed; run files qwen-qwen3.8-2.4t-a95b_mc_route-digitalocean_maxtok52000.json5 / qwen-qwen3.8-2.4t-a95b_route-digitalocean_maxtok52000.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.774038,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+79+38=161/208. Roblox LLM Leaderboard entry 'Qwen 3.8 2.4T-A95B' (https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B); no reasoning setting listed; run files qwen-qwen3.8-2.4t-a95b_mc_route-digitalocean_maxtok52000.json5 / qwen-qwen3.8-2.4t-a95b_route-digitalocean_maxtok52000.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-fable-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.969667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2909/3000 correct (noAnswer=27). Roblox LLM Leaderboard entry 'Claude Fable 5 (Medium)' (https://www.anthropic.com/claude/fable); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-fable-5_mc_effort-medium.json5 / claude-fable-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-fable-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.850962,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+84+49=177/208. Roblox LLM Leaderboard entry 'Claude Fable 5 (Medium)' (https://www.anthropic.com/claude/fable); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-fable-5_mc_effort-medium.json5 / claude-fable-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "robloxqa-v2-mc",
    "score": 0.979333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2938/3000 correct (noAnswer=4). Roblox LLM Leaderboard entry 'Claude Fable 5.1 (Medium)' (https://www.anthropic.com/claude/fable); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-fable-5-1_mc_effort-medium.json5 / claude-fable-5-1_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "luau-leetcode-full-pass",
    "score": 0.831731,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+80+49=173/208. Roblox LLM Leaderboard entry 'Claude Fable 5.1 (Medium)' (https://www.anthropic.com/claude/fable); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-fable-5-1_mc_effort-medium.json5 / claude-fable-5-1_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.895,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2685/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Claude Haiku 4.5 (Extended Thinking)' (https://www.anthropic.com/claude/haiku); effort/setting as labelled 'Extended Thinking' (the only setting listed for this model); run files claude-haiku-4-5-20251001_mc_thinking.json5 / claude-haiku-4-5-20251001_thinking.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.596154,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+54+27=124/208. Roblox LLM Leaderboard entry 'Claude Haiku 4.5 (Extended Thinking)' (https://www.anthropic.com/claude/haiku); effort/setting as labelled 'Extended Thinking' (the only setting listed for this model); run files claude-haiku-4-5-20251001_mc_thinking.json5 / claude-haiku-4-5-20251001_thinking.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "robloxqa-v2-mc",
    "score": 0.966667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2900/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Claude Opus 4.8 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-4-8_mc_effort-medium.json5 / claude-opus-4-8_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "luau-leetcode-full-pass",
    "score": 0.798077,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+75+47=166/208. Roblox LLM Leaderboard entry 'Claude Opus 4.8 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-4-8_mc_effort-medium.json5 / claude-opus-4-8_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-opus-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.975,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2925/3000 correct (noAnswer=2). Roblox LLM Leaderboard entry 'Claude Opus 5 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-5_mc_effort-medium.json5 / claude-opus-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-opus-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.822115,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+80+47=171/208. Roblox LLM Leaderboard entry 'Claude Opus 5 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-5_mc_effort-medium.json5 / claude-opus-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.939667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2819/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Claude Sonnet 5 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-sonnet-5_mc_effort-medium.json5 / claude-sonnet-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.682692,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 41+69+32=142/208. Roblox LLM Leaderboard entry 'Claude Sonnet 5 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-sonnet-5_mc_effort-medium.json5 / claude-sonnet-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "robloxqa-v2-mc",
    "score": 0.892,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2676/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Deepseek R1 0528' (https://huggingface.co/deepseek-ai/DeepSeek-R1-0528); no reasoning setting listed; run files deepseek-deepseek-r1-0528_mc_route-siliconflow-fp8.json5 / deepseek-deepseek-r1-0528_route-siliconflow-fp8.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "luau-leetcode-full-pass",
    "score": 0.596154,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 41+61+22=124/208. Roblox LLM Leaderboard entry 'Deepseek R1 0528' (https://huggingface.co/deepseek-ai/DeepSeek-R1-0528); no reasoning setting listed; run files deepseek-deepseek-r1-0528_mc_route-siliconflow-fp8.json5 / deepseek-deepseek-r1-0528_route-siliconflow-fp8.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "robloxqa-v2-mc",
    "score": 0.918333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2755/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Deepseek V4 Flash 0731' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731); no reasoning setting listed; run files deepseek-deepseek-v4-flash-0731_mc_route-siliconflow-fp8_maxtok128000.json5 / deepseek-deepseek-v4-flash-0731_route-siliconflow-fp8_maxtok128000.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "luau-leetcode-full-pass",
    "score": 0.778846,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+76+43=162/208. Roblox LLM Leaderboard entry 'Deepseek V4 Flash 0731' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731); no reasoning setting listed; run files deepseek-deepseek-v4-flash-0731_mc_route-siliconflow-fp8_maxtok128000.json5 / deepseek-deepseek-v4-flash-0731_route-siliconflow-fp8_maxtok128000.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "robloxqa-v2-mc",
    "score": 0.926,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2778/3000 correct (noAnswer=1). Roblox LLM Leaderboard entry 'Deepseek V4 Pro 0813 (High)' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813); effort/setting as labelled 'High' (the only setting listed for this model); run files deepseek-deepseek-v4-pro-0813_mc_effort-high_route-deepseek.json5 / deepseek-deepseek-v4-pro-0813_effort-high_route-deepseek.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "luau-leetcode-full-pass",
    "score": 0.822115,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+80+48=171/208. Roblox LLM Leaderboard entry 'Deepseek V4 Pro 0813 (High)' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813); effort/setting as labelled 'High' (the only setting listed for this model); run files deepseek-deepseek-v4-pro-0813_mc_effort-high_route-deepseek.json5 / deepseek-deepseek-v4-pro-0813_effort-high_route-deepseek.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "robloxqa-v2-mc",
    "score": 0.928333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2785/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemini 2.5 Pro (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-2.5-pro); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gemini-2.5-pro_mc_effort-medium.json5 / gemini-2.5-pro_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "luau-leetcode-full-pass",
    "score": 0.653846,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+64+29=136/208. Roblox LLM Leaderboard entry 'Gemini 2.5 Pro (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-2.5-pro); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gemini-2.5-pro_mc_effort-medium.json5 / gemini-2.5-pro_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "robloxqa-v2-mc",
    "score": 0.965333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2896/3000 correct (noAnswer=5). Roblox LLM Leaderboard entry 'Gemini 3.1 Pro (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gemini-3.1-pro-preview_mc_effort-medium.json5 / gemini-3.1-pro-preview_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "luau-leetcode-full-pass",
    "score": 0.826923,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+79+50=172/208. Roblox LLM Leaderboard entry 'Gemini 3.1 Pro (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gemini-3.1-pro-preview_mc_effort-medium.json5 / gemini-3.1-pro-preview_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "robloxqa-v2-mc",
    "score": 0.967667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2903/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemini 3.7 Flash (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.7-flash); effort/setting as labelled 'Medium' (the only setting listed for this model); run files google-gemini-3.7-flash_mc_route-google-vertex-global.json5 / google-gemini-3.7-flash_route-google-vertex-global.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "luau-leetcode-full-pass",
    "score": 0.798077,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+79+44=166/208. Roblox LLM Leaderboard entry 'Gemini 3.7 Flash (Medium)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.7-flash); effort/setting as labelled 'Medium' (the only setting listed for this model); run files google-gemini-3.7-flash_mc_route-google-vertex-global.json5 / google-gemini-3.7-flash_route-google-vertex-global.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "robloxqa-v2-mc",
    "score": 0.965333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2896/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemini 3.8 Flash (High)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash); effort/setting as labelled 'High' (the only setting listed for this model); run files google-gemini-3.8-flash_mc_effort-high.json5 / google-gemini-3.8-flash_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "luau-leetcode-full-pass",
    "score": 0.846154,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+85+47=176/208. Roblox LLM Leaderboard entry 'Gemini 3.8 Flash (High)' (https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash); effort/setting as labelled 'High' (the only setting listed for this model); run files google-gemini-3.8-flash_mc_effort-high.json5 / google-gemini-3.8-flash_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-3-27b",
    "bench": "robloxqa-v2-mc",
    "score": 0.845333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2536/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemma 3 27B' (https://huggingface.co/google/gemma-3-27b-it); no reasoning setting listed; run files google-gemma-3-27b-it_mc.json5 / google-gemma-3-27b-it.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-3-27b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.163462,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 15+13+6=34/208. Roblox LLM Leaderboard entry 'Gemma 3 27B' (https://huggingface.co/google/gemma-3-27b-it); no reasoning setting listed; run files google-gemma-3-27b-it_mc.json5 / google-gemma-3-27b-it.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "robloxqa-v2-mc",
    "score": 0.909333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2728/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemma 4 26B-A4B' (https://huggingface.co/google/gemma-4-26B-A4B); no reasoning setting listed; run files google-gemma-4-26b-a4b-it_mc_route-novita-bf16.json5 / google-gemma-4-26b-a4b-it_route-novita-bf16.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.389423,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 27+44+10=81/208. Roblox LLM Leaderboard entry 'Gemma 4 26B-A4B' (https://huggingface.co/google/gemma-4-26B-A4B); no reasoning setting listed; run files google-gemma-4-26b-a4b-it_mc_route-novita-bf16.json5 / google-gemma-4-26b-a4b-it_route-novita-bf16.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-4-31b",
    "bench": "robloxqa-v2-mc",
    "score": 0.925667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2777/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Gemma 4 31B' (https://huggingface.co/google/gemma-4-31B); no reasoning setting listed; run files google-gemma-4-31b-it_mc.json5 / google-gemma-4-31b-it.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gemma-4-31b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.552885,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 37+52+26=115/208. Roblox LLM Leaderboard entry 'Gemma 4 31B' (https://huggingface.co/google/gemma-4-31B); no reasoning setting listed; run files google-gemma-4-31b-it_mc.json5 / google-gemma-4-31b-it.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "robloxqa-v2-mc",
    "score": 0.787333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2362/3000 correct (noAnswer=4). Roblox LLM Leaderboard entry 'Llama 3.1 8B Instruct' (https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct); no reasoning setting listed; run files meta-llama-llama-3.1-8b-instruct_mc.json5 / meta-llama-llama-3.1-8b-instruct.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.081731,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 11+4+2=17/208. Roblox LLM Leaderboard entry 'Llama 3.1 8B Instruct' (https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct); no reasoning setting listed; run files meta-llama-llama-3.1-8b-instruct_mc.json5 / meta-llama-llama-3.1-8b-instruct.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "kimi-k3",
    "bench": "robloxqa-v2-mc",
    "score": 0.964667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2894/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Kimi K3 (High)' (https://www.kimi.com/blog/kimi-k3); effort/setting as labelled 'High' (the only setting listed for this model); run files moonshotai-kimi-k3_mc_effort-high.json5 / moonshotai-kimi-k3_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "kimi-k3",
    "bench": "luau-leetcode-full-pass",
    "score": 0.778846,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 42+75+45=162/208. Roblox LLM Leaderboard entry 'Kimi K3 (High)' (https://www.kimi.com/blog/kimi-k3); effort/setting as labelled 'High' (the only setting listed for this model); run files moonshotai-kimi-k3_mc_effort-high.json5 / moonshotai-kimi-k3_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "nemotron-3-5-lightning",
    "bench": "robloxqa-v2-mc",
    "score": 0.876667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2630/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Nemotron 3.5 Lightning' (https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16); no reasoning setting listed; run files nvidia-nemotron-3.5-lightning_mc_route-coreweave-bf16.json5 / nvidia-nemotron-3.5-lightning_route-coreweave-bf16.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "nemotron-3-5-lightning",
    "bench": "luau-leetcode-full-pass",
    "score": 0.528846,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 35+52+23=110/208. Roblox LLM Leaderboard entry 'Nemotron 3.5 Lightning' (https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16); no reasoning setting listed; run files nvidia-nemotron-3.5-lightning_mc_route-coreweave-bf16.json5 / nvidia-nemotron-3.5-lightning_route-coreweave-bf16.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.943667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2831/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5 (Medium)' (https://developers.openai.com/api/docs/models/gpt-5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5_mc_effort-medium.json5 / gpt-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.788462,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+78+42=164/208. Roblox LLM Leaderboard entry 'GPT 5 (Medium)' (https://developers.openai.com/api/docs/models/gpt-5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5_mc_effort-medium.json5 / gpt-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "robloxqa-v2-mc",
    "score": 0.89,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2670/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5.4 nano (High)' (https://developers.openai.com/api/docs/models/gpt-5.4-nano); effort/setting as labelled 'High' (the only setting listed for this model); run files gpt-5.4-nano_mc_effort-high.json5 / gpt-5.4-nano_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "luau-leetcode-full-pass",
    "score": 0.649038,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 39+64+32=135/208. Roblox LLM Leaderboard entry 'GPT 5.4 nano (High)' (https://developers.openai.com/api/docs/models/gpt-5.4-nano); effort/setting as labelled 'High' (the only setting listed for this model); run files gpt-5.4-nano_mc_effort-high.json5 / gpt-5.4-nano_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.975333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2926/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5.5 (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.5_mc_effort-medium.json5 / gpt-5.5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.850962,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+82+52=177/208. Roblox LLM Leaderboard entry 'GPT 5.5 (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.5_mc_effort-medium.json5 / gpt-5.5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "robloxqa-v2-mc",
    "score": 0.957667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2873/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5.6 Luna (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-luna); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-luna_mc_effort-medium.json5 / gpt-5.6-luna_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "luau-leetcode-full-pass",
    "score": 0.725962,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+74+33=151/208. Roblox LLM Leaderboard entry 'GPT 5.6 Luna (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-luna); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-luna_mc_effort-medium.json5 / gpt-5.6-luna_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "robloxqa-v2-mc",
    "score": 0.979333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2938/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5.6 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-sol_mc_effort-medium.json5 / gpt-5.6-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "luau-leetcode-full-pass",
    "score": 0.846154,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 42+81+53=176/208. Roblox LLM Leaderboard entry 'GPT 5.6 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-sol_mc_effort-medium.json5 / gpt-5.6-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "robloxqa-v2-mc",
    "score": 0.957667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2873/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 5.6 Terra (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-terra); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-terra_mc_effort-medium.json5 / gpt-5.6-terra_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "luau-leetcode-full-pass",
    "score": 0.8125,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+76+49=169/208. Roblox LLM Leaderboard entry 'GPT 5.6 Terra (Medium)' (https://developers.openai.com/api/docs/models/gpt-5.6-terra); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-5.6-terra_mc_effort-medium.json5 / gpt-5.6-terra_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "robloxqa-v2-mc",
    "score": 0.887,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2661/3000 correct (noAnswer=1). Roblox LLM Leaderboard entry 'GPT OSS 20B' (https://huggingface.co/openai/gpt-oss-20b); no reasoning setting listed; run files openai-gpt-oss-20b_mc.json5 / openai-gpt-oss-20b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.524038,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 38+55+16=109/208. Roblox LLM Leaderboard entry 'GPT OSS 20B' (https://huggingface.co/openai/gpt-oss-20b); no reasoning setting listed; run files openai-gpt-oss-20b_mc.json5 / openai-gpt-oss-20b.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "grok-4-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.962333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2887/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Grok 4.5 (Medium)' (https://docs.x.ai/developers/models/grok-4.5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files x-ai-grok-4.5_mc_effort-medium.json5 / x-ai-grok-4.5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "grok-4-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.783654,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+79+40=163/208. Roblox LLM Leaderboard entry 'Grok 4.5 (Medium)' (https://docs.x.ai/developers/models/grok-4.5); effort/setting as labelled 'Medium' (the only setting listed for this model); run files x-ai-grok-4.5_mc_effort-medium.json5 / x-ai-grok-4.5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "grok-4-6",
    "bench": "robloxqa-v2-mc",
    "score": 0.967667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2903/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Grok 4.6 (Medium)' (https://docs.x.ai/developers/models/grok-4.6); effort/setting as labelled 'Medium' (the only setting listed for this model); run files x-ai-grok-4.6_mc_effort-medium_route-xai-zdr.json5 / x-ai-grok-4.6_effort-medium_route-xai-zdr.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "grok-4-6",
    "bench": "luau-leetcode-full-pass",
    "score": 0.836538,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 45+81+48=174/208. Roblox LLM Leaderboard entry 'Grok 4.6 (Medium)' (https://docs.x.ai/developers/models/grok-4.6); effort/setting as labelled 'Medium' (the only setting listed for this model); run files x-ai-grok-4.6_mc_effort-medium_route-xai-zdr.json5 / x-ai-grok-4.6_effort-medium_route-xai-zdr.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "robloxqa-v2-mc",
    "score": 0.831333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2494/3000 correct (noAnswer=91). Roblox LLM Leaderboard entry 'GLM 4.7 Flash' (https://huggingface.co/zai-org/GLM-4.7-Flash); no reasoning setting listed; run files z-ai-glm-4.7-flash_mc.json5 / z-ai-glm-4.7-flash.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "luau-leetcode-full-pass",
    "score": 0.149038,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 20+11+0=31/208. Roblox LLM Leaderboard entry 'GLM 4.7 Flash' (https://huggingface.co/zai-org/GLM-4.7-Flash); no reasoning setting listed; run files z-ai-glm-4.7-flash_mc.json5 / z-ai-glm-4.7-flash.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-2",
    "bench": "robloxqa-v2-mc",
    "score": 0.89,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2670/3000 correct (noAnswer=180). Roblox LLM Leaderboard entry 'GLM 5.2 (High)' (https://huggingface.co/zai-org/GLM-5.2); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.2_mc_effort-high_route-baidu-fp8.json5 / z-ai-glm-5.2_effort-high_route-baidu-fp8.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-2",
    "bench": "luau-leetcode-full-pass",
    "score": 0.740385,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 43+76+35=154/208. Roblox LLM Leaderboard entry 'GLM 5.2 (High)' (https://huggingface.co/zai-org/GLM-5.2); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.2_mc_effort-high_route-baidu-fp8.json5 / z-ai-glm-5.2_effort-high_route-baidu-fp8.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-3",
    "bench": "robloxqa-v2-mc",
    "score": 0.939333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2818/3000 correct (noAnswer=3). Roblox LLM Leaderboard entry 'GLM 5.3 (High)' (https://huggingface.co/zai-org/GLM-5.3); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.3_mc_effort-high.json5 / z-ai-glm-5.3_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-3",
    "bench": "luau-leetcode-full-pass",
    "score": 0.697115,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 40+67+38=145/208. Roblox LLM Leaderboard entry 'GLM 5.3 (High)' (https://huggingface.co/zai-org/GLM-5.3); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.3_mc_effort-high.json5 / z-ai-glm-5.3_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "robloxqa-v2-mc",
    "score": 0.946,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2838/3000 correct (noAnswer=1). Roblox LLM Leaderboard entry 'GLM 5.3 Flash (High)' (https://huggingface.co/zai-org/GLM-5.3-Flash); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.3-flash_mc_effort-high.json5 / z-ai-glm-5.3-flash_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "luau-leetcode-full-pass",
    "score": 0.6875,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "source_id": "roblox-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 44+72+27=143/208. Roblox LLM Leaderboard entry 'GLM 5.3 Flash (High)' (https://huggingface.co/zai-org/GLM-5.3-Flash); effort/setting as labelled 'High' (the only setting listed for this model); run files z-ai-glm-5.3-flash_mc_effort-high.json5 / z-ai-glm-5.3-flash_effort-high.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43."
  },
  {
    "model": "deepseek-r1",
    "bench": "webapp1k-react-pass1",
    "score": 0.927,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-01-22",
    "note": "Entry 'deepseek-r1' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. results.csv was last updated 2025-04-08 and the entry first appeared 2025-01-22, so this is the January 2025 R1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-v3",
    "bench": "webapp1k-react-pass1",
    "score": 0.8723,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-12-28",
    "note": "Entry 'deepseek-v3' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-12-28 (before V3-0324), so this is the December 2024 V3. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gemma-3-27b",
    "bench": "webapp1k-react-pass1",
    "score": 0.655,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-03-16",
    "note": "Entry 'gemma-3-27b' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "webapp1k-react-pass1",
    "score": 0.3545,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-12-28",
    "note": "Entry 'qwen2p5-72b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (2000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "webapp1k-react-pass1",
    "score": 0.334,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-01-14",
    "note": "Entry 'llama-v3p3-70b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "webapp1k-react-pass1",
    "score": 0.323,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Entry 'deepseek-r1-distill-qwen-32b' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "webapp1k-react-pass1",
    "score": 0.302,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'llama-v3p1-405b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "webapp1k-react-pass1",
    "score": 0.2512,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'llama-v3p1-8b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "webapp1k-react-pass1",
    "score": 0.1027,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'llama-v3p1-70b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "llama-4-maverick",
    "bench": "webapp1k-react-pass1",
    "score": 0.769,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Entry 'llama4-maverick' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Llama 4 Maverick 17B-128E Instruct (served build not published). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "webapp1k-react-pass1",
    "score": 0.668,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-01-14",
    "note": "Entry 'qwen2p5-coder-32b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gpt-5",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.793794,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-09-17",
    "note": "Entry 'gpt-5' in results_duo.csv: mean per-problem Successes/Runs over 999 problems (999 runs), reproducing the Space's pass@1. One problem missing from the published file (999/1000). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.784,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Entry 'claude-sonnet-4-5' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gpt-5-1",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.765,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-11-21",
    "note": "Entry 'gpt-5.1' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.763,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Entry 'claude-opus-4-5-20251101' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.75,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-09-20",
    "note": "Entry 'gpt-oss-120b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Served precision/provider not published. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "o4-mini",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.726,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Entry 'o4-mini' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.707,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-11-09",
    "note": "Entry 'kimi-k2-thinking' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gpt-4-1",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.688,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-04-16",
    "note": "Entry 'gpt-4.1' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-r1",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.687,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-01-22",
    "note": "Entry 'deepseek-r1' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Listed separately from 'deepseek-r1-2025-05-28' and first published 2025-01-22, so this is the January 2025 R1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.685,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-05-31",
    "note": "Entry 'deepseek-r1-2025-05-28' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.65,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-12-04",
    "note": "Entry 'deepseek-v3.2' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Listed separately from 'deepseek-v3.2-exp' and first published 2025-12-04, so this is the December 2025 V3.2 release. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.616,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-03-25",
    "note": "Entry 'deepseek-v3-0324' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.612,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-11-02",
    "note": "Entry 'qwen3-coder-480b-a35b-instruct' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-v3",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.585,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2024-12-28",
    "note": "Entry 'deepseek-v3' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Listed separately from 'deepseek-v3-0324' and first published 2024-12-28, so this is the December 2024 V3. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "glm-4-6",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.578,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-11-19",
    "note": "Entry 'glm-4p6' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.329,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Entry 'qwen3-235b-a22b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Original April 2025 hybrid checkpoint name (no 2507 suffix), first published 2025-05-01; thinking mode not stated. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "qwen3-32b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.243,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Entry 'qwen3-32b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "gemma-3-27b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.198,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-03-14",
    "note": "Entry 'gemma-3-27b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.087,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "source_id": "webapp1k-leaderboard",
    "by": "independent",
    "date": "2025-03-11",
    "note": "Entry 'deepseek-r1-distill-qwen-32b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0."
  },
  {
    "model": "claude-opus-5-5",
    "bench": "mideind-icelandic-average",
    "score": 0.708666,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Opus 5.5' (anthropic/claude-opus-5-5), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high']. Published 'average' (Knowledge 0.54585, Language Proficiency 0.830779). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-6-astra",
    "bench": "mideind-icelandic-average",
    "score": 0.702515,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-6 Astra' (openai/gpt-6-astra), thinking='high', gen_kwargs={\"max_completion_tokens\": 32000, \"reasoning_effort\": \"high\", \"allowed_openai_params\": [\"reasoning_effort\"]}; highest of the listed settings ['high', 'low']. Published 'average' (Knowledge 0.5132, Language Proficiency 0.8445). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "mideind-icelandic-average",
    "score": 0.694172,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Fable 5.1' (anthropic/claude-fable-5-1), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high']. Published 'average' (Knowledge 0.525292, Language Proficiency 0.820831). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "claude-opus-5",
    "bench": "mideind-icelandic-average",
    "score": 0.68627,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Opus 5' (anthropic/claude-opus-5), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.496945, Language Proficiency 0.828264). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "mideind-icelandic-average",
    "score": 0.665418,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 3.1 Pro (Preview)' (gemini/gemini-3.1-pro-preview), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'low']. Published 'average' (Knowledge 0.495921, Language Proficiency 0.792542). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-6-sol",
    "bench": "mideind-icelandic-average",
    "score": 0.656274,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-6 Sol' (openai/gpt-6-sol), thinking='high', gen_kwargs={\"max_completion_tokens\": 32000, \"reasoning_effort\": \"high\", \"allowed_openai_params\": [\"reasoning_effort\"]}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.442644, Language Proficiency 0.816497). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "mideind-icelandic-average",
    "score": 0.652881,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-5.6 Sol' (openai/gpt-5.6-sol), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.472464, Language Proficiency 0.788194). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "mideind-icelandic-average",
    "score": 0.649668,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 3.7 Flash' (gemini/gemini-3.7-flash), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'low']. Published 'average' (Knowledge 0.482071, Language Proficiency 0.775366). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "mideind-icelandic-average",
    "score": 0.638808,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Opus 4.8' (anthropic/claude-opus-4-8), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.438468, Language Proficiency 0.789064). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "mideind-icelandic-average",
    "score": 0.634764,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 3.8 Flash' (gemini/gemini-3.8-flash), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'low']. Published 'average' (Knowledge 0.490192, Language Proficiency 0.743192). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "mideind-icelandic-average",
    "score": 0.622247,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 3.5 Flash' (gemini/gemini-3.5-flash), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.459254, Language Proficiency 0.744493). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-6-luna",
    "bench": "mideind-icelandic-average",
    "score": 0.571384,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-6 Luna' (openai/gpt-6-luna), thinking='high', gen_kwargs={\"max_completion_tokens\": 32000, \"reasoning_effort\": \"high\", \"allowed_openai_params\": [\"reasoning_effort\"]}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.376226, Language Proficiency 0.717753). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "mideind-icelandic-average",
    "score": 0.551725,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Sonnet 5' (anthropic/claude-sonnet-5), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.347428, Language Proficiency 0.704947). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "mideind-icelandic-average",
    "score": 0.539251,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-5.6 Luna' (openai/gpt-5.6-luna), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.351636, Language Proficiency 0.679963). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "mideind-icelandic-average",
    "score": 0.531822,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-5.6 Terra' (openai/gpt-5.6-terra), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.349448, Language Proficiency 0.668602). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "mideind-icelandic-average",
    "score": 0.522084,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 2.5 Pro' (gemini/gemini-2.5-pro), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'minimal']. Published 'average' (Knowledge 0.391655, Language Proficiency 0.619906). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "mideind-icelandic-average",
    "score": 0.511622,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-08",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'DeepSeek V4 Pro' (together_ai/deepseek-ai/DeepSeek-V4-Pro-0813), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"reasoning\": {\"enabled\": true}}; highest of the listed settings ['off', 'on']. Published 'average' (Knowledge 0.351534, Language Proficiency 0.631688). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "mideind-icelandic-average",
    "score": 0.495686,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 3.5 Flash-Lite' (gemini/gemini-3.5-flash-lite), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.319032, Language Proficiency 0.628178). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-5-2",
    "bench": "mideind-icelandic-average",
    "score": 0.469714,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-5.2' (openai/gpt-5.2), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.317561, Language Proficiency 0.583829). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "mideind-icelandic-average",
    "score": 0.44136,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-23",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Qwen3.8 2.4T-A95B' (together_ai/Qwen/Qwen3.8-2.4T-A95B), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"reasoning\": {\"enabled\": true}}; highest of the listed settings ['on']. Published 'average' (Knowledge 0.258022, Language Proficiency 0.578863). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "kimi-k3",
    "bench": "mideind-icelandic-average",
    "score": 0.407066,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Kimi K3' (together_ai/moonshotai/Kimi-K3), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"reasoning\": {\"enabled\": true}}; highest of the listed settings ['off', 'on']. Published 'average' (Knowledge 0.317775, Language Proficiency 0.474035). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "minimax-m3",
    "bench": "mideind-icelandic-average",
    "score": 0.312964,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'MiniMax-M3' (together_ai/MiniMaxAI/MiniMax-M3), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"reasoning\": {\"enabled\": true}}; highest of the listed settings ['on']. Published 'average' (Knowledge 0.217277, Language Proficiency 0.384729). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "mideind-icelandic-average",
    "score": 0.278617,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GLM-5.3-Flash' (together_ai/zai-org/GLM-5.3-Flash), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"reasoning\": {\"enabled\": true}}; highest of the listed settings ['on']. Published 'average' (Knowledge 0.229802, Language Proficiency 0.315228). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gemma-4-31b",
    "bench": "mideind-icelandic-average",
    "score": 0.207289,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-09",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemma 4 31B IT' (hosted_vllm/google/gemma-4-31B-it), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"chat_template_kwargs\": {\"enable_thinking\": true}, \"skip_special_tokens\": false}}; highest of the listed settings ['on']. Published 'average' (Knowledge 0.155854, Language Proficiency 0.245866). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "mideind-icelandic-average",
    "score": 0.156837,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "source_id": "mideind-icelandic-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Haiku 4.5' (anthropic/claude-haiku-4-5), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.109377, Language Proficiency 0.192432). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b."
  },
  {
    "model": "gpt-5",
    "bench": "soohak-mini-avg3",
    "score": 0.7222,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-5'; decoding per Table 4: temp 1.0, reasoning effort medium. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 72.22%."
  },
  {
    "model": "gpt-5",
    "bench": "soohak-challenge-avg3",
    "score": 0.2637,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-5'; decoding per Table 4: temp 1.0, reasoning effort medium. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 26.37%."
  },
  {
    "model": "gpt-5-mini",
    "bench": "soohak-mini-avg3",
    "score": 0.6714,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-5-Mini'; decoding per Table 4: temp 1.0, reasoning effort medium. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 67.14%."
  },
  {
    "model": "gpt-5-mini",
    "bench": "soohak-challenge-avg3",
    "score": 0.1882,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-5-Mini'; decoding per Table 4: temp 1.0, reasoning effort medium. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 18.82%."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "soohak-mini-avg3",
    "score": 0.5138,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Claude-Opus-4.5'; decoding per Table 4: temp 0.6, extended thinking. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 51.38%."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "soohak-challenge-avg3",
    "score": 0.1039,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Claude-Opus-4.5'; decoding per Table 4: temp 0.6, extended thinking. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 10.39%."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "soohak-mini-avg3",
    "score": 0.4088,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Claude-Sonnet-4.5'; decoding per Table 4: temp 0.6, extended thinking. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 40.88%."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "soohak-challenge-avg3",
    "score": 0.0569,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Claude-Sonnet-4.5'; decoding per Table 4: temp 0.6, extended thinking. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 5.69%."
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "soohak-mini-avg3",
    "score": 0.5622,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Qwen3-235B-A22B-thinking-2507'; decoding per Table 4: temp 0.6, thinking variant, default context. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 56.22%."
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "soohak-challenge-avg3",
    "score": 0.0804,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Qwen3-235B-A22B-thinking-2507'; decoding per Table 4: temp 0.6, thinking variant, default context. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 8.04%."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "soohak-mini-avg3",
    "score": 0.6102,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-OSS-120B'; decoding per Table 4: temp 1.0, reasoning effort medium (main-results setting; Table 6 higher-effort runs report Pass@3 only). Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 61.02%."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "soohak-challenge-avg3",
    "score": 0.1127,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GPT-OSS-120B'; decoding per Table 4: temp 1.0, reasoning effort medium (main-results setting; Table 6 higher-effort runs report Pass@3 only). Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 11.27%."
  },
  {
    "model": "kimi-k2-5",
    "bench": "soohak-mini-avg3",
    "score": 0.6607,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Kimi-2.5'; decoding per Table 4: temp 0.6, thinking variant. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 66.07%."
  },
  {
    "model": "kimi-k2-5",
    "bench": "soohak-challenge-avg3",
    "score": 0.1387,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'Kimi-2.5'; decoding per Table 4: temp 0.6, thinking variant. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 13.87%."
  },
  {
    "model": "glm-5",
    "bench": "soohak-mini-avg3",
    "score": 0.6311,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GLM-5'; decoding per Table 4: temp 0.6, thinking variant. Authors' own run. Column SOOHAK-Mini (n=702) Avg@3 = 63.11%."
  },
  {
    "model": "glm-5",
    "bench": "soohak-challenge-avg3",
    "score": 0.0961,
    "source": "https://arxiv.org/html/2605.09063v3",
    "source_id": "paper-2605.09063v3",
    "by": "independent",
    "date": "2026-05-19",
    "note": "Soohak (arXiv:2605.09063v3, CC BY-SA 4.0) Table 2, row 'GLM-5'; decoding per Table 4: temp 0.6, thinking variant. Authors' own run. Column SOOHAK Challenge (n=340) Avg@3 = 9.61%."
  },
  {
    "model": "gpt-5-5",
    "bench": "mathconstraint-329-notools",
    "score": 0.669,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'gpt-5.5': Correct=220, Accuracy=66.9%. Reasoning config per Table 10: reasoning effort=high. Authors' own run."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "mathconstraint-329-notools",
    "score": 0.426,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'claude-opus-4.7': Correct=140, Accuracy=42.6%. Reasoning config per Table 10: reasoning max_tokens=8000. Authors' own run."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "mathconstraint-329-notools",
    "score": 0.185,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'claude-4.6-sonnet': Correct=61, Accuracy=18.5%. Reasoning config per Table 10: reasoning: none. Authors' own run."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "mathconstraint-329-notools",
    "score": 0.517,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'gemini-3.1-pro': Correct=170, Accuracy=51.7%. Reasoning config per Table 10: reasoning effort=high. Authors' own run."
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "mathconstraint-329-notools",
    "score": 0.249,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'gemini-3.1-flash-lite': Correct=82, Accuracy=24.9%. Reasoning config per Table 10: reasoning effort=medium. Authors' own run."
  },
  {
    "model": "qwen3-6-plus",
    "bench": "mathconstraint-329-notools",
    "score": 0.474,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'qwen3.6-plus': Correct=156, Accuracy=47.4%. Reasoning config per Table 10: reasoning effort=high. Authors' own run."
  },
  {
    "model": "kimi-k2-6",
    "bench": "mathconstraint-329-notools",
    "score": 0.359,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'kimi-k2.6': Correct=118, Accuracy=35.9%. Reasoning config per Table 10: reasoning effort=high. Authors' own run."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "mathconstraint-329-notools",
    "score": 0.395,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'gpt-oss-120b': Correct=130, Accuracy=39.5%. Reasoning config per Table 10: reasoning config: none (provider default). Authors' own run."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "mathconstraint-329-notools",
    "score": 0.085,
    "source": "https://arxiv.org/html/2605.08498v1",
    "source_id": "paper-2605.08498v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "MathConstraint (arXiv:2605.08498v1, CC BY 4.0) Table 2 (no tools, full 329-instance release), row 'llama-3.3-70b-instruct': Correct=28, Accuracy=8.5%. Reasoning config per Table 10: no reasoning. Authors' own run."
  },
  {
    "model": "gpt-5-4",
    "bench": "dseffi-bench-pass",
    "score": 0.669,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'GPT-5.4', Overall Pass = 66.9% (greedy T=0.0). Identifier per Appendix C.2: gpt-5.4 (proprietary API). Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "dseffi-bench-pass",
    "score": 0.622,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'GPT-5.4-mini', Overall Pass = 62.2% (greedy T=0.0). Identifier per Appendix C.2: gpt-5.4-mini (proprietary API). Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "dseffi-bench-pass",
    "score": 0.613,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Claude-Opus-4.6', Overall Pass = 61.3% (greedy T=0.0). Identifier per Appendix C.2: claude-opus-4-6 (proprietary API). Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "dseffi-bench-pass",
    "score": 0.597,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Claude-Sonnet-4.6', Overall Pass = 59.7% (greedy T=0.0). Identifier per Appendix C.2: claude-sonnet-4-6 (proprietary API). Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "dseffi-bench-pass",
    "score": 0.561,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'DeepSeek-V3.2', Overall Pass = 56.1% (greedy T=0.0). Identifier per Appendix C.2: deepseek-ai/DeepSeek-V3.2 served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "kimi-k2-5",
    "bench": "dseffi-bench-pass",
    "score": 0.402,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Kimi-K2.5', Overall Pass = 40.2% (greedy T=0.0). Identifier per Appendix C.2: moonshotai/Kimi-K2.5 served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "minimax-m2-5",
    "bench": "dseffi-bench-pass",
    "score": 0.428,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'MiniMax-M2.5', Overall Pass = 42.8% (greedy T=0.0). Identifier per Appendix C.2: MiniMaxAI/MiniMax-M2.5 served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "dseffi-bench-pass",
    "score": 0.576,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Qwen3.5-397B/A17B', Overall Pass = 57.6% (greedy T=0.0). Identifier per Appendix C.2: Qwen/Qwen3.5-397B-A17B served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "qwen3-coder-next",
    "bench": "dseffi-bench-pass",
    "score": 0.505,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Qwen3-Coder-Next', Overall Pass = 50.5% (greedy T=0.0). Identifier per Appendix C.2: Qwen/Qwen3-Coder-Next served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "dseffi-bench-pass",
    "score": 0.341,
    "source": "https://arxiv.org/html/2608.30248v1",
    "source_id": "paper-2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Qwen2.5-Coder-32B', Overall Pass = 34.1% (greedy T=0.0). Identifier per Appendix C.2: Qwen/Qwen2.5-Coder-32B-Instruct served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "finsheet-bench-acc",
    "score": 0.824,
    "source": "https://arxiv.org/html/2603.07316v1",
    "source_id": "paper-2603.07316v1",
    "by": "independent",
    "date": "2026-03-07",
    "note": "FinSheet-Bench (arXiv:2603.07316v1, CC BY 4.0) Table 4, row 'Gemini 3.1 Pro', Acc. = 82.4% (coverage 100%). Setting: provider default reasoning. Zero-shot, no prompt engineering. Authors' own run."
  },
  {
    "model": "gpt-5-2",
    "bench": "finsheet-bench-acc",
    "score": 0.804,
    "source": "https://arxiv.org/html/2603.07316v1",
    "source_id": "paper-2603.07316v1",
    "by": "independent",
    "date": "2026-03-07",
    "note": "FinSheet-Bench (arXiv:2603.07316v1, CC BY 4.0) Table 4, row 'GPT-5.2 (reasoning)', Acc. = 80.4% (coverage 100%). Setting: OpenAI reasoning_effort enabled (level not stated); the paper's no-reasoning GPT-5.2 row (57.7%) not imported. Zero-shot, no prompt engineering. Authors' own run."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "finsheet-bench-acc",
    "score": 0.802,
    "source": "https://arxiv.org/html/2603.07316v1",
    "source_id": "paper-2603.07316v1",
    "by": "independent",
    "date": "2026-03-07",
    "note": "FinSheet-Bench (arXiv:2603.07316v1, CC BY 4.0) Table 4, row 'Claude Opus 4.6 (thinking)', Acc. = 80.2% (coverage 100%). Setting: Anthropic extended thinking enabled; the paper's non-thinking row (66.7%) not imported. Zero-shot, no prompt engineering. Authors' own run."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "finsheet-bench-acc",
    "score": 0.788,
    "source": "https://arxiv.org/html/2603.07316v1",
    "source_id": "paper-2603.07316v1",
    "by": "independent",
    "date": "2026-03-07",
    "note": "FinSheet-Bench (arXiv:2603.07316v1, CC BY 4.0) Table 4, row 'Gemini 2.5 Pro', Acc. = 78.8% (coverage 100%). Setting: provider default reasoning. Zero-shot, no prompt engineering. Authors' own run."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.578,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Claude Opus 4.5' / agent 'Terminus 2': resolution rate 57.8% ± 2.5% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-5-2",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.54,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-5.2' / agent 'Terminus 2': resolution rate 54.0% ± 2.9% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.428,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Claude Sonnet 4.5' / agent 'Terminus 2': resolution rate 42.8% ± 2.8% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.357,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Kimi K2 Thinking' / agent 'Terminus 2': resolution rate 35.7% ± 2.8% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-5",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.352,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-5' / agent 'Terminus 2': resolution rate 35.2% ± 3.1% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.326,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Gemini 2.5 Pro' / agent 'Terminus 2': resolution rate 32.6% ± 3.0% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "minimax-m2",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.3,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'MiniMax M2' / agent 'Terminus 2': resolution rate 30.0% ± 2.7% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.283,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Claude Haiku 4.5' / agent 'Terminus 2': resolution rate 28.3% ± 2.9% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "glm-4-6",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.245,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GLM 4.6' / agent 'Terminus 2': resolution rate 24.5% ± 2.4% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-5-mini",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.24,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-5-Mini' / agent 'Terminus 2': resolution rate 24.0% ± 2.5% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.239,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Qwen 3 Coder 480B' / agent 'Terminus 2': resolution rate 23.9% ± 2.8% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.187,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-OSS-120B' / agent 'Terminus 2': resolution rate 18.7% ± 2.7% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.169,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'Gemini 2.5 Flash' / agent 'Terminus 2': resolution rate 16.9% ± 2.4% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-5-nano",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.079,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-5-Nano' / agent 'Terminus 2': resolution rate 7.9% ± 1.9% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "terminal-bench-2-terminus2",
    "score": 0.031,
    "source": "https://arxiv.org/html/2601.11868v1",
    "source_id": "paper-2601.11868v1",
    "by": "independent",
    "date": "2026-01-17",
    "note": "Terminal-Bench paper (arXiv:2601.11868v1, CC BY 4.0) Table 2, row 'GPT-OSS-20B' / agent 'Terminus 2': resolution rate 3.1% ± 1.5% (95% CI). Reasoning effort: provider default ('medium' for OpenAI and Anthropic models); first-party APIs for closed models. Benchmark authors' own trials; paper co-authors include task contributors from several companies, but runs were not vendor-reported."
  },
  {
    "model": "gpt-5-5",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.211,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'GPT-5.5', Overall Pass Rate = 21.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-4",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.205,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'GPT-5.4', Overall Pass Rate = 20.5%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.151,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Claude Opus 4.7', Overall Pass Rate = 15.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.141,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Gemini 3.1 Pro', Overall Pass Rate = 14.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.141,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Claude Opus 4.6', Overall Pass Rate = 14.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "qwen3-7-max",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.118,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Qwen3.7 Max', Overall Pass Rate = 11.8%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.099,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Claude Sonnet 4.6', Overall Pass Rate = 9.9%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "kimi-k2-6",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.092,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Kimi K2.6', Overall Pass Rate = 9.2%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "mimo-v2-5",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.086,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'MIMO v2.5', Overall Pass Rate = 8.6%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "qwen3-6-plus",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.086,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Qwen3.6 Plus', Overall Pass Rate = 8.6%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "grok-4-3",
    "bench": "ale-openclaw-overall-pass",
    "score": 0.043,
    "source": "https://arxiv.org/html/2606.05405v2",
    "source_id": "paper-2606.05405v2",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Agents' Last Exam (arXiv:2606.05405v2, CC BY 4.0) Table 1, panel 'LLM Model Comparison (fixed OpenClaw + GUI-as-Tool)', row 'Grok 4.3', Overall Pass Rate = 4.3%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-1",
    "bench": "medmarks-v-winrate",
    "score": 0.6395,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'GPT-5.1 (med)', win rate = 0.6395. Settings per Table 8: reasoning effort medium; temp 1.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "medmarks-v-winrate",
    "score": 0.6258,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Claude Sonnet 4.5', win rate = 0.6258. Settings per Table 8: temp 0.7. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gpt-5-2",
    "bench": "medmarks-v-winrate",
    "score": 0.6236,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'GPT-5.2 (med)', win rate = 0.6236. Settings per Table 8: reasoning effort medium; temp 1.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "glm-4-7",
    "bench": "medmarks-v-winrate",
    "score": 0.6199,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'GLM 4.7 FP8', win rate = 0.6199. Settings per Table 8: official FP8 weights; temp 1.0, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "medmarks-v-winrate",
    "score": 0.6032,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 235B-A22B Thinking', win rate = 0.6032. Settings per Table 8: temp 0.6, top_p 0.95, top_k 20; Thinking-named Qwen3 235B checkpoint (only released as -Thinking-2507). Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "medmarks-v-winrate",
    "score": 0.5888,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 Next 80B-A3B Thinking', win rate = 0.5888. Settings per Table 8: temp 0.6, top_p 0.95, top_k 20. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "medmarks-v-winrate",
    "score": 0.5865,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'gpt-oss 120b (high)', win rate = 0.5865. Settings per Table 8: reasoning effort high (highest of high/med/low rows); temp 1.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "minimax-m2",
    "bench": "medmarks-v-winrate",
    "score": 0.5708,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'MiniMax M2', win rate = 0.5708. Settings per Table 8: temp 1.0, top_p 0.95, top_k 40. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "medmarks-v-winrate",
    "score": 0.5687,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 Next 80B-A3B Instruct', win rate = 0.5687. Settings per Table 8: temp 0.7, top_p 0.8, top_k 20. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "medmarks-v-winrate",
    "score": 0.5587,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 30B-A3B Thinking', win rate = 0.5587. Settings per Table 8: temp 0.6, top_p 0.95, top_k 20; unquantized row (FP8/8-bit/4-bit rows not imported); Thinking-named 30B-A3B checkpoint (only released as -Thinking-2507). Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "glm-4-5-air",
    "bench": "medmarks-v-winrate",
    "score": 0.541,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'GLM 4.5 Air', win rate = 0.541. Settings per Table 8: temp 0.6, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-14b",
    "bench": "medmarks-v-winrate",
    "score": 0.5367,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 14B (Thinking)', win rate = 0.5367. Settings per Table 8: hybrid Qwen3-14B in thinking mode; temp 0.6, top_p 0.95, top_k 20. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "medmarks-v-winrate",
    "score": 0.5364,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Llama 3.3 70B Instruct', win rate = 0.5364. Settings per Table 8: temp 0.7, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "medmarks-v-winrate",
    "score": 0.5361,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'gpt-oss 20b (high)', win rate = 0.5361. Settings per Table 8: reasoning effort high (highest of high/med/low rows); temp 1.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "medmarks-v-winrate",
    "score": 0.5317,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 30B-A3B Instruct', win rate = 0.5317. Settings per Table 8: temp 0.7, top_p 0.8, top_k 20; unquantized row; Instruct-named 30B-A3B checkpoint (only released as -Instruct-2507). Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "medmarks-v-winrate",
    "score": 0.5114,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Nemotron Nano V3 30B-A3B', win rate = 0.5114. Settings per Table 8: Nemotron 3 Nano 30B-A3B per Table 8; temp 1.0, top_p 1.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "medmarks-v-winrate",
    "score": 0.5067,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Olmo 3.1 32B Think', win rate = 0.5067. Settings per Table 8: temp 0.6, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-8b",
    "bench": "medmarks-v-winrate",
    "score": 0.5019,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 8B (Thinking)', win rate = 0.5019. Settings per Table 8: hybrid Qwen3-8B in thinking mode; temp 0.6, top_p 0.95, top_k 20. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen2-5-32b",
    "bench": "medmarks-v-winrate",
    "score": 0.5003,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen2.5 32B Instruct', win rate = 0.5003. Settings per Table 8: sampling not listed in Table 8. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "medmarks-v-winrate",
    "score": 0.4891,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Qwen3 4B Thinking', win rate = 0.4891. Settings per Table 8: temp 0.6, top_p 0.95, top_k 20; Thinking-named Qwen3 4B checkpoint (only released as -Thinking-2507). Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "phi-4-reasoning",
    "bench": "medmarks-v-winrate",
    "score": 0.4815,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Phi 4 Reasoning', win rate = 0.4815. Settings per Table 8: temp 0.8, top_p 0.95, top_k 50. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "medmarks-v-winrate",
    "score": 0.4807,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 14B Reasoning', win rate = 0.4807. Settings per Table 8: temp 0.7, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-14b",
    "bench": "medmarks-v-winrate",
    "score": 0.476,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 14B Instruct', win rate = 0.476. Settings per Table 8: temp 0.1, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gemma-3-27b",
    "bench": "medmarks-v-winrate",
    "score": 0.461,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Gemma 3 27B', win rate = 0.461. Settings per Table 8: temp 1.0, top_p 0.95, top_k 60. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "medmarks-v-winrate",
    "score": 0.4452,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Granite 4.0H Small', win rate = 0.4452. Settings per Table 8: temp 0.0. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-8b",
    "bench": "medmarks-v-winrate",
    "score": 0.4442,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 8B Instruct', win rate = 0.4442. Settings per Table 8: temp 0.1, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gemma-3-12b",
    "bench": "medmarks-v-winrate",
    "score": 0.4379,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Gemma 3 12B', win rate = 0.4379. Settings per Table 8: temp 1.0, top_p 0.95, top_k 60. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "medmarks-v-winrate",
    "score": 0.4312,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 8B Reasoning', win rate = 0.4312. Settings per Table 8: temp 0.7, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "medmarks-v-winrate",
    "score": 0.413,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Olmo 3 7B Think', win rate = 0.413. Settings per Table 8: temp 0.6, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "medmarks-v-winrate",
    "score": 0.3967,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Llama 3.1 8B Instruct', win rate = 0.3967. Settings per Table 8: temp 0.7, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-3b",
    "bench": "medmarks-v-winrate",
    "score": 0.3927,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 3B Instruct', win rate = 0.3927. Settings per Table 8: temp 0.1, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "medmarks-v-winrate",
    "score": 0.3741,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Ministral 3 3B Reasoning', win rate = 0.3741. Settings per Table 8: temp 0.7, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "smollm3-3b",
    "bench": "medmarks-v-winrate",
    "score": 0.3548,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'SmolLM3 3B', win rate = 0.3548. Settings per Table 8: temp 0.6, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "medmarks-v-winrate",
    "score": 0.3503,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Olmo 3 7B Instruct', win rate = 0.3503. Settings per Table 8: temp 0.6, top_p 0.95. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gemma-3-4b",
    "bench": "medmarks-v-winrate",
    "score": 0.3214,
    "source": "https://arxiv.org/html/2605.01417v1",
    "source_id": "paper-2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks (arXiv:2605.01417v1, CC BY 4.0) Table 4 (all models by mean win rate on Medmarks-V), row 'Gemma 3 4B', win rate = 0.3214. Settings per Table 8: temp 1.0, top_p 0.95, top_k 60. Pool-relative score over the paper's 71 configurations. Authors' own run."
  },
  {
    "model": "gpt-5-2",
    "bench": "hle-verified-text-full",
    "score": 0.433,
    "source": "https://arxiv.org/html/2602.13964v4",
    "source_id": "paper-2602.13964v4",
    "by": "independent",
    "date": "2026-08-17",
    "note": "HLE-Verified (arXiv:2602.13964v4, CC BY 4.0) Table 2, row 'GPT-5.2-High', column 'Revised: HLE-Verified Full' Acc = 43.3% (text-only subset). Evaluated by the paper authors (Qwen Team, Alibaba Group), a third party to this model's maker; effort as labelled in the row (GPT-5.2 high; others default)."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "hle-verified-text-full",
    "score": 0.388,
    "source": "https://arxiv.org/html/2602.13964v4",
    "source_id": "paper-2602.13964v4",
    "by": "independent",
    "date": "2026-08-17",
    "note": "HLE-Verified (arXiv:2602.13964v4, CC BY 4.0) Table 2, row 'Claude-Opus4.5', column 'Revised: HLE-Verified Full' Acc = 38.8% (text-only subset). Evaluated by the paper authors (Qwen Team, Alibaba Group), a third party to this model's maker; effort as labelled in the row (GPT-5.2 high; others default)."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "hle-verified-text-full",
    "score": 0.468,
    "source": "https://arxiv.org/html/2602.13964v4",
    "source_id": "paper-2602.13964v4",
    "by": "independent",
    "date": "2026-08-17",
    "note": "HLE-Verified (arXiv:2602.13964v4, CC BY 4.0) Table 2, row 'Claude-Opus4.6', column 'Revised: HLE-Verified Full' Acc = 46.8% (text-only subset). Evaluated by the paper authors (Qwen Team, Alibaba Group), a third party to this model's maker; effort as labelled in the row (GPT-5.2 high; others default)."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "hle-verified-text-full",
    "score": 0.364,
    "source": "https://arxiv.org/html/2602.13964v4",
    "source_id": "paper-2602.13964v4",
    "by": "independent",
    "date": "2026-08-17",
    "note": "HLE-Verified (arXiv:2602.13964v4, CC BY 4.0) Table 2, row 'Deepseek-V3.2', column 'Revised: HLE-Verified Full' Acc = 36.4% (text-only subset). Evaluated by the paper authors (Qwen Team, Alibaba Group), a third party to this model's maker; effort as labelled in the row (GPT-5.2 high; others default)."
  },
  {
    "model": "gpt-5-5",
    "bench": "token-games-elo",
    "score": 1167,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'gpt-5.5', TTG Elo = 1167. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "token-games-elo",
    "score": 1129,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'gemini-3.1-pro-preview', TTG Elo = 1129. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "token-games-elo",
    "score": 1100,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'claude-opus-4-7', TTG Elo = 1100. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "token-games-elo",
    "score": 1068,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'gpt-5.4-mini', TTG Elo = 1068. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "grok-4-20",
    "bench": "token-games-elo",
    "score": 1057,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'grok-4.20-0309-reasoning', TTG Elo = 1057. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "token-games-elo",
    "score": 1043,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'claude-sonnet-4-6', TTG Elo = 1043. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "token-games-elo",
    "score": 1007,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'deepseek-v3.2-thinking', TTG Elo = 1007. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "token-games-elo",
    "score": 1000,
    "source": "https://arxiv.org/html/2602.17831v2",
    "source_id": "paper-2602.17831v2",
    "by": "independent",
    "date": "2026-05-15",
    "note": "The Token Games (arXiv:2602.17831v2, CC BY-SA 4.0) Table 1, row 'claude-haiku-4-5', TTG Elo = 1000. All models run in default settings with the highest thinking/reasoning mode enabled. Authors' own duels; other columns of Table 1 are copied leaderboard values and were not imported."
  },
  {
    "model": "claude-fable-5",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.744,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Claude Fable 5', Src 'ours', T1 Pair = 74.4%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.702,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'GPT-5.6 Sol', Src 'ours', T1 Pair = 70.2%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-5",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.699,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'GPT-5.5', Src 'ours', T1 Pair = 69.9%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gemma-4-31b",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.699,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Gemma-4-31B-it', Src 'ours', T1 Pair = 69.9%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.676,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Claude Opus 4.8', Src 'ours', T1 Pair = 67.6%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.662,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Qwen3.6-35B-A3B', Src 'ours', T1 Pair = 66.2%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.611,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Qwen3.5-35B-A3B', Src 'ours', T1 Pair = 61.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gemma-4-e4b",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.526,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Gemma-4-E4B-it', Src 'ours', T1 Pair = 52.6%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "qwen3-5-9b",
    "bench": "sciclaimeval-t1-pair-dev",
    "score": 0.483,
    "source": "https://arxiv.org/html/2609.00654v1",
    "source_id": "paper-2609.00654v1",
    "by": "independent",
    "date": "2026-09-01",
    "note": "SciTrue (arXiv:2609.00654v1, CC BY 4.0) Table 3 (single-model dev results, honest per-sample), row 'Qwen3.5-9B', Src 'ours', T1 Pair = 48.3%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-opus-4-7",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.625,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'Claude Opus 4.7', Pass Rate = 62.5% (resolved rate excluding harness errors = 64.8%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "kimi-k2-6",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.575,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'Kimi K2.6', Pass Rate = 57.5% (resolved rate excluding harness errors = 60.2%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "glm-5-1",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.57,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'GLM 5.1', Pass Rate = 57.0% (resolved rate excluding harness errors = 58.5%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.55,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'Gemini 3.1 Pro', Pass Rate = 55.0% (resolved rate excluding harness errors = 57.0%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "gpt-5-5",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.535,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'GPT-5.5', Pass Rate = 53.5% (resolved rate excluding harness errors = 56.6%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "minimax-m2-7",
    "bench": "terminalworld-verified-terminus2",
    "score": 0.49,
    "source": "https://arxiv.org/html/2605.22535v1",
    "source_id": "paper-2605.22535v1",
    "by": "independent",
    "date": "2026-05-21",
    "note": "TerminalWorld (arXiv:2605.22535v1, CC BY 4.0) Table 1, row 'MiniMax M2.7', Pass Rate = 49.0% (resolved rate excluding harness errors = 50.8%, not used). Terminus-2 scaffold; provider-default reasoning (e.g. medium for GPT-5.5, high for Claude Opus 4.7), temperature 0.7. Authors' own run."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "mathnet-solve-test-micro",
    "score": 0.784,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gemini-3.1-pro-preview' in group 'LMMs with reasoning', Micro Avg. = 78.4%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5",
    "bench": "mathnet-solve-test-micro",
    "score": 0.693,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gpt-5' in group 'LMMs with reasoning', Micro Avg. = 69.3%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-mini",
    "bench": "mathnet-solve-test-micro",
    "score": 0.57,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gpt-5-mini' in group 'LMMs with reasoning', Micro Avg. = 57.0%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "claude-opus-4-6",
    "bench": "mathnet-solve-test-micro",
    "score": 0.457,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'claude-opus-4.6' in group 'LMMs with reasoning', Micro Avg. = 45.7%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-5-nano",
    "bench": "mathnet-solve-test-micro",
    "score": 0.422,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gpt-5-nano' in group 'LMMs with reasoning', Micro Avg. = 42.2%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "mathnet-solve-test-micro",
    "score": 0.411,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gemini-2.5-flash' in group 'LMMs with reasoning', Micro Avg. = 41.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "mathnet-solve-test-micro",
    "score": 0.401,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'DeepSeek-V3.2' in group 'LLMs (text-only), non-reasoning group', Micro Avg. = 40.1%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "gpt-4-1",
    "bench": "mathnet-solve-test-micro",
    "score": 0.214,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'gpt-4.1' in group 'LMMs (text + images)', Micro Avg. = 21.4%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "llama-4-maverick",
    "bench": "mathnet-solve-test-micro",
    "score": 0.147,
    "source": "https://arxiv.org/html/2604.18584v2",
    "source_id": "paper-2604.18584v2",
    "by": "independent",
    "date": "2026-07-29",
    "note": "MathNet (arXiv:2604.18584v2, CC BY 4.0) Table 3 (MathNet-Solve-Test, 6,400 problems), row 'Llama-4-Maverick-17B*' in group 'LMMs (text + images)', Micro Avg. = 14.7%. Reasoning effort not stated. Authors' own run."
  },
  {
    "model": "mistral-large-2",
    "bench": "aragen-v3-3c3h",
    "score": 0.5202,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-27",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Mistral-Large-Instruct-2411'; 3C3H Score 0.5202 (95% CI 0.472271-0.567139, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250927_014123; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "aragen-v3-3c3h",
    "score": 0.1054,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.2-1B-Instruct'; 3C3H Score 0.1054 (95% CI 0.083633-0.127229, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_031424; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "aragen-v3-3c3h",
    "score": 0.2494,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.1-8B-Instruct'; 3C3H Score 0.2494 (95% CI 0.212662-0.288195, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_052822; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "aragen-v3-3c3h",
    "score": 0.5212,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.3-70B-Instruct'; 3C3H Score 0.5212 (95% CI 0.473977-0.568326, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_192611; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "aragen-v3-3c3h",
    "score": 0.5,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.1-70B-Instruct'; 3C3H Score 0.5 (95% CI 0.45104-0.549846, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_225335; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-2-3b",
    "bench": "aragen-v3-3c3h",
    "score": 0.1499,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.2-3B-Instruct'; 3C3H Score 0.1499 (95% CI 0.123371-0.176182, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250930_032633; RM_Thinking_Traces=None."
  },
  {
    "model": "smollm3-3b",
    "bench": "aragen-v3-3c3h",
    "score": 0.1624,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'HuggingFaceTB/SmolLM3-3B'; 3C3H Score 0.1624 (95% CI 0.127752-0.199606, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20250930_230403; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen2-5-14b",
    "bench": "aragen-v3-3c3h",
    "score": 0.369,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen2.5-14B-Instruct'; 3C3H Score 0.369 (95% CI 0.325793-0.412214, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251001_074608; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen3-14b",
    "bench": "aragen-v3-3c3h",
    "score": 0.2974,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen3-14B'; 3C3H Score 0.2974 (95% CI 0.254374-0.342339, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251001_115246; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "aragen-v3-3c3h",
    "score": 0.4892,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen2.5-72B-Instruct'; 3C3H Score 0.4892 (95% CI 0.443162-0.534217, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251001_174705; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen3-32b",
    "bench": "aragen-v3-3c3h",
    "score": 0.3518,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-02",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen3-32B'; 3C3H Score 0.3518 (95% CI 0.306584-0.398707, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251002_042140; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen3-8b",
    "bench": "aragen-v3-3c3h",
    "score": 0.2655,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-02",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen3-8B'; 3C3H Score 0.2655 (95% CI 0.223424-0.309371, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251002_083731; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen2-5-7b",
    "bench": "aragen-v3-3c3h",
    "score": 0.2625,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-02",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen2.5-7B-Instruct'; 3C3H Score 0.2625 (95% CI 0.226466-0.299214, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251002_105513; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen3-0-6b",
    "bench": "aragen-v3-3c3h",
    "score": 0.0157,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-02",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen3-0.6B'; 3C3H Score 0.0157 (95% CI 0.005873-0.027521, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251002_134703; RM_Thinking_Traces=None."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "aragen-v3-3c3h",
    "score": 0.7817,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-sonnet-4-5-20250929'; 3C3H Score 0.7817 (95% CI 0.749364-0.812339, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251128_115648; RM_Thinking_Traces=None."
  },
  {
    "model": "claude-opus-4-5",
    "bench": "aragen-v3-3c3h",
    "score": 0.8029,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-opus-4-5-20251101'; 3C3H Score 0.8029 (95% CI 0.775848-0.82904, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251128_150307; RM_Thinking_Traces=None."
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "aragen-v3-3c3h",
    "score": 0.658,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-haiku-4-5-20251001'; 3C3H Score 0.658 (95% CI 0.617848-0.697495, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251128_173352; RM_Thinking_Traces=None."
  },
  {
    "model": "gpt-5",
    "bench": "aragen-v3-3c3h",
    "score": 0.8425,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-29",
    "note": "Inception AraGen-v3 leaderboard entry 'gpt-5-2025-08-07'; 3C3H Score 0.8425 (95% CI 0.816691-0.866445, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251129_122123; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown)."
  },
  {
    "model": "o3",
    "bench": "aragen-v3-3c3h",
    "score": 0.8219,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Inception AraGen-v3 leaderboard entry 'o3-2025-04-16'; 3C3H Score 0.8219 (95% CI 0.788867-0.852421, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251130_120313; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown)."
  },
  {
    "model": "o4-mini",
    "bench": "aragen-v3-3c3h",
    "score": 0.706,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Inception AraGen-v3 leaderboard entry 'o4-mini-2025-04-16'; 3C3H Score 0.706 (95% CI 0.662453-0.747551, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251130_173427; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown)."
  },
  {
    "model": "phi-4",
    "bench": "aragen-v3-3c3h",
    "score": 0.2998,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-12-19",
    "note": "Inception AraGen-v3 leaderboard entry 'microsoft/phi-4'; 3C3H Score 0.2998 (95% CI 0.257718-0.342725, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251219_180254; RM_Thinking_Traces=None."
  },
  {
    "model": "gpt-4-1",
    "bench": "aragen-v3-3c3h",
    "score": 0.7454,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-02-04",
    "note": "Inception AraGen-v3 leaderboard entry 'gpt-4.1'; 3C3H Score 0.7454 (95% CI 0.710696-0.777784, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260204_120723; RM_Thinking_Traces=None. Entry uses the unversioned alias 'gpt-4.1'; OpenAI has published only one GPT-4.1 snapshot (gpt-4.1-2025-04-14)."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aragen-v3-3c3h",
    "score": 0.3061,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-02-08",
    "note": "Inception AraGen-v3 leaderboard entry 'openai/gpt-oss-20b'; 3C3H Score 0.3061 (95% CI 0.265431-0.34564, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260208_184056; RM_Thinking_Traces=None."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aragen-v3-3c3h",
    "score": 0.4323,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-02-12",
    "note": "Inception AraGen-v3 leaderboard entry 'openai/gpt-oss-120b'; 3C3H Score 0.4323 (95% CI 0.3931-0.471118, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260212_132744; RM_Thinking_Traces=True."
  },
  {
    "model": "glm-5",
    "bench": "aragen-v3-3c3h",
    "score": 0.4601,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-03-16",
    "note": "Inception AraGen-v3 leaderboard entry 'zai-org/GLM-5'; 3C3H Score 0.4601 (95% CI 0.414564-0.506047, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260316_030628; RM_Thinking_Traces=False."
  },
  {
    "model": "magistral-small-2509",
    "bench": "aragen-v3-3c3h",
    "score": 0.3231,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-03",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Magistral-Small-2509'; 3C3H Score 0.3231 (95% CI 0.282409-0.36514, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260403_123855; RM_Thinking_Traces=False."
  },
  {
    "model": "gemma-4-31b",
    "bench": "aragen-v3-3c3h",
    "score": 0.7271,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-03",
    "note": "Inception AraGen-v3 leaderboard entry 'google/gemma-4-31B-it'; 3C3H Score 0.7271 (95% CI 0.695046-0.757718, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260403_124446; RM_Thinking_Traces=False."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "aragen-v3-3c3h",
    "score": 0.6735,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-03",
    "note": "Inception AraGen-v3 leaderboard entry 'google/gemma-4-26B-A4B-it'; 3C3H Score 0.6735 (95% CI 0.637262-0.707262, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260403_125341; RM_Thinking_Traces=False."
  },
  {
    "model": "mistral-small-4",
    "bench": "aragen-v3-3c3h",
    "score": 0.3385,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-04",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Mistral-Small-4-119B-2603'; 3C3H Score 0.3385 (95% CI 0.297587-0.379634, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260404_070113; RM_Thinking_Traces=False."
  },
  {
    "model": "o3",
    "bench": "hindigen-v1-3c3h",
    "score": 0.8556,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'o3-2025-04-16'; 3C3H Score 0.8556 (95% CI 0.82142-0.887473, n=266); judge claude-3.5-sonnet; 266/279 entries answered. No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "o4-mini",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7552,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'o4-mini-2025-04-16'; 3C3H Score 0.7552 (95% CI 0.711022-0.796897, n=263); judge claude-3.5-sonnet; 263/279 entries answered. No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "gpt-4-1",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7337,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4.1-2025-04-14'; 3C3H Score 0.7337 (95% CI 0.689291-0.777313, n=264); judge claude-3.5-sonnet; 264/279 entries answered. No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "hindigen-v1-3c3h",
    "score": 0.6502,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4.1-mini-2025-04-14'; 3C3H Score 0.6502 (95% CI 0.599881-0.697676, n=266); judge claude-3.5-sonnet; 266/279 entries answered. No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "gemma-3-27b",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5032,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'google/gemma-3-27b-it'; 3C3H Score 0.5032 (95% CI 0.455522-0.549159, n=267); judge claude-3.5-sonnet; 267/279 entries answered. No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-5-7b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.502,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2.5-7B-Instruct'; 'Ar Prompt-lvl' 0.502 (Ar instruction-level 0.555, En prompt-level 0.682 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-5-32b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.604,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2.5-32B-Instruct'; 'Ar Prompt-lvl' 0.604 (Ar instruction-level 0.669, En prompt-level 0.776 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.673,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2.5-72B-Instruct'; 'Ar Prompt-lvl' 0.673 (Ar instruction-level 0.741, En prompt-level 0.835 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.416,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.1-8B-Instruct'; 'Ar Prompt-lvl' 0.416 (Ar instruction-level 0.505, En prompt-level 0.706 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.448,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.1-70B-Instruct'; 'Ar Prompt-lvl' 0.448 (Ar instruction-level 0.502, En prompt-level 0.829 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.507,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.1-405B-Instruct'; 'Ar Prompt-lvl' 0.507 (Ar instruction-level 0.57, En prompt-level 0.837 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.111,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.2-1B-Instruct'; 'Ar Prompt-lvl' 0.111 (Ar instruction-level 0.155, En prompt-level 0.459 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-2-3b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.191,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.2-3B-Instruct'; 'Ar Prompt-lvl' 0.191 (Ar instruction-level 0.285, En prompt-level 0.635 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.582,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-3.3-70B-Instruct'; 'Ar Prompt-lvl' 0.582 (Ar instruction-level 0.631, En prompt-level 0.882 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7253,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-08-24",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'meta-llama/Meta-Llama-3.1-70B-Instruct', model path meta-llama/Meta-Llama-3.1-70B-Instruct; eq-bench_v2_pl score 72.53 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-08-24 22:23:40.",
    "raw_value": 72.53,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-7b",
    "bench": "polish-eq-bench-v2",
    "score": 0.5858,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-7B-Instruct', model path Qwen/Qwen2.5-7B-Instruct; eq-bench_v2_pl score 58.58 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-01 11:49:28.",
    "raw_value": 58.58,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-14b",
    "bench": "polish-eq-bench-v2",
    "score": 0.6958,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-14B-Instruct', model path Qwen/Qwen2.5-14B-Instruct; eq-bench_v2_pl score 69.58 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-01 12:01:56.",
    "raw_value": 69.58,
    "scale_divisor": 100
  },
  {
    "model": "phi-4",
    "bench": "polish-eq-bench-v2",
    "score": 0.6437,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'phi-4', model path microsoft/phi-4; eq-bench_v2_pl score 64.37 (divided by 100); 157.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-01 12:19:38.",
    "raw_value": 64.37,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-32b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7115,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-32B-Instruct', model path Qwen/Qwen2.5-32B-Instruct; eq-bench_v2_pl score 71.15 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 14:08:52.",
    "raw_value": 71.15,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-72b",
    "bench": "polish-eq-bench-v2",
    "score": 0.6889,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-72B-Instruct', model path Qwen/Qwen2.5-72B-Instruct; eq-bench_v2_pl score 68.89 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 14:25:32.",
    "raw_value": 68.89,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-2-1b",
    "bench": "polish-eq-bench-v2",
    "score": 0.2059,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Llama-3.2-1B-Instruct', model path meta-llama/Llama-3.2-1B-Instruct; eq-bench_v2_pl score 20.59 (divided by 100); 148.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 16:35:24.",
    "raw_value": 20.59,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-3-70b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7286,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Llama-3.3-70B-Instruct', model path meta-llama/Llama-3.3-70B-Instruct; eq-bench_v2_pl score 72.86 (divided by 100); 166.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 16:59:31.",
    "raw_value": 72.86,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-2-3b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4646,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Llama-3.2-3B-Instruct', model path meta-llama/Llama-3.2-3B-Instruct; eq-bench_v2_pl score 46.46 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 17:14:17.",
    "raw_value": 46.46,
    "scale_divisor": 100
  },
  {
    "model": "phi-4-mini",
    "bench": "polish-eq-bench-v2",
    "score": 0.5082,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-03",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Phi-4-mini-instruct', model path microsoft/Phi-4-mini-instruct; eq-bench_v2_pl score 50.82 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-03 13:20:03.",
    "raw_value": 50.82,
    "scale_divisor": 100
  },
  {
    "model": "mistral-large-2",
    "bench": "polish-eq-bench-v2",
    "score": 0.7729,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-07",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Mistral-Large-Instruct-2411', model path mistralai/Mistral-Large-Instruct-2411; eq-bench_v2_pl score 77.29 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-07 12:17:17.",
    "raw_value": 77.29,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "polish-eq-bench-v2",
    "score": 0.7346,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-12-24",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'DeepSeek-V3-0324', model path deepseek-ai/DeepSeek-V3-0324; eq-bench_v2_pl score 73.46 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2025-12-24 01:21:41.",
    "raw_value": 73.46,
    "scale_divisor": 100
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4582,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2026-03-26",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'NVIDIA-Nemotron-3-Nano-30B-A3B-BF16', model path nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16; eq-bench_v2_pl score 45.82 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2026-03-26 22:30:53.",
    "raw_value": 45.82,
    "scale_divisor": 100
  },
  {
    "model": "mistral-large-2",
    "bench": "cptub-speakleash-avg",
    "score": 0.800832,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Large-Instruct-2411'; Space Average = mean(Sentiment 4.327, Language understanding 3.975, Phraseology 3.990, Tricky questions 3.725) = 4.0042 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.004161,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-1-70b",
    "bench": "cptub-speakleash-avg",
    "score": 0.724908,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Meta-Llama-3.1-70B-Instruct'; Space Average = mean(Sentiment 4.327, Language understanding 3.910, Phraseology 3.250, Tricky questions 3.011) = 3.6245 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.62454,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-1-8b",
    "bench": "cptub-speakleash-avg",
    "score": 0.602336,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Meta-Llama-3.1-8B-Instruct'; Space Average = mean(Sentiment 3.974, Language understanding 3.380, Phraseology 2.580, Tricky questions 2.112) = 3.0117 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.01168,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-2-1b",
    "bench": "cptub-speakleash-avg",
    "score": 0.38372,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Llama-3.2-1B-Instruct'; Space Average = mean(Sentiment 3.077, Language understanding 1.735, Phraseology 2.340, Tricky questions 0.522) = 1.9186 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.918599,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-2-3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.399526,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Llama-3.2-3B-Instruct'; Space Average = mean(Sentiment 2.756, Language understanding 2.295, Phraseology 1.720, Tricky questions 1.219) = 1.9976 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.997628,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-72b",
    "bench": "cptub-speakleash-avg",
    "score": 0.789296,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-72B-Instruct'; Space Average = mean(Sentiment 4.077, Language understanding 3.970, Phraseology 3.930, Tricky questions 3.809) = 3.9465 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.946478,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-32b",
    "bench": "cptub-speakleash-avg",
    "score": 0.7502,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-32B-Instruct'; Space Average = mean(Sentiment 3.814, Language understanding 3.565, Phraseology 4.035, Tricky questions 3.590) = 3.7510 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.750998,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-3-70b",
    "bench": "cptub-speakleash-avg",
    "score": 0.728814,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Llama-3.3-70B-Instruct'; Space Average = mean(Sentiment 4.295, Language understanding 3.865, Phraseology 3.040, Tricky questions 3.376) = 3.6441 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.644069,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-14b",
    "bench": "cptub-speakleash-avg",
    "score": 0.709117,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-14B-Instruct'; Space Average = mean(Sentiment 3.910, Language understanding 3.565, Phraseology 3.370, Tricky questions 3.337) = 3.5456 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.545584,
    "scale_divisor": 5
  },
  {
    "model": "phi-4",
    "bench": "cptub-speakleash-avg",
    "score": 0.660883,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'microsoft/phi-4'; Space Average = mean(Sentiment 3.718, Language understanding 3.540, Phraseology 3.235, Tricky questions 2.725) = 3.3044 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.304417,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-7b",
    "bench": "cptub-speakleash-avg",
    "score": 0.613098,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-7B-Instruct'; Space Average = mean(Sentiment 3.558, Language understanding 3.025, Phraseology 3.095, Tricky questions 2.584) = 3.0655 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.06549,
    "scale_divisor": 5
  },
  {
    "model": "phi-4-mini",
    "bench": "cptub-speakleash-avg",
    "score": 0.433534,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'microsoft/Phi-4-mini-instruct'; Space Average = mean(Sentiment 2.692, Language understanding 2.430, Phraseology 2.245, Tricky questions 1.303) = 2.1677 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.16767,
    "scale_divisor": 5
  },
  {
    "model": "deepseek-v3",
    "bench": "cptub-speakleash-avg",
    "score": 0.804637,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'deepseek-ai/DeepSeek-V3 (API)'; Space Average = mean(Sentiment 4.359, Language understanding 4.220, Phraseology 3.525, Tricky questions 3.989) = 4.0232 on a 0-5 judge scale, divided by 5. Served through an API; the HF id deepseek-ai/DeepSeek-V3 is the December 2024 checkpoint. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.023185,
    "scale_divisor": 5
  },
  {
    "model": "deepseek-r1",
    "bench": "cptub-speakleash-avg",
    "score": 0.827508,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'deepseek-ai/DeepSeek-R1 (API)'; Space Average = mean(Sentiment 4.487, Language understanding 4.345, Phraseology 3.600, Tricky questions 4.118) = 4.1375 on a 0-5 judge scale, divided by 5. Served through an API; the HF id deepseek-ai/DeepSeek-R1 is the January 2025 checkpoint (not R1-0528). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.137539,
    "scale_divisor": 5
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "cptub-speakleash-avg",
    "score": 0.805822,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'deepseek-ai/DeepSeek-V3-0324 (API)'; Space Average = mean(Sentiment 4.359, Language understanding 4.195, Phraseology 3.540, Tricky questions 4.022) = 4.0291 on a 0-5 judge scale, divided by 5. Served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.029112,
    "scale_divisor": 5
  },
  {
    "model": "gemma-3-27b",
    "bench": "cptub-speakleash-avg",
    "score": 0.761096,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'google/gemma-3-27b-it (API)'; Space Average = mean(Sentiment 3.878, Language understanding 3.785, Phraseology 4.025, Tricky questions 3.534) = 3.8055 on a 0-5 judge scale, divided by 5. Served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.805478,
    "scale_divisor": 5
  },
  {
    "model": "llama-4-maverick",
    "bench": "cptub-speakleash-avg",
    "score": 0.786723,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 (API)'; Space Average = mean(Sentiment 4.391, Language understanding 4.110, Phraseology 3.475, Tricky questions 3.758) = 3.9336 on a 0-5 judge scale, divided by 5. Meta's official FP8 release served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.933613,
    "scale_divisor": 5
  },
  {
    "model": "llama-4-scout",
    "bench": "cptub-speakleash-avg",
    "score": 0.749929,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Llama-4-Scout-17B-16E-Instruct (API)'; Space Average = mean(Sentiment 4.103, Language understanding 3.805, Phraseology 3.900, Tricky questions 3.191) = 3.7496 on a 0-5 judge scale, divided by 5. Served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.749644,
    "scale_divisor": 5
  },
  {
    "model": "deepseek-v3-2",
    "bench": "cptub-speakleash-avg",
    "score": 0.827838,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'deepseek-ai/DeepSeek-V3.2 (API)'; Space Average = mean(Sentiment 4.455, Language understanding 4.360, Phraseology 3.545, Tricky questions 4.197) = 4.1392 on a 0-5 judge scale, divided by 5. Served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.139189,
    "scale_divisor": 5
  },
  {
    "model": "deepseek-v3-1",
    "bench": "cptub-speakleash-avg",
    "score": 0.805193,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'deepseek-ai/DeepSeek-V3.1 (API)'; Space Average = mean(Sentiment 4.423, Language understanding 4.335, Phraseology 3.475, Tricky questions 3.871) = 4.0260 on a 0-5 judge scale, divided by 5. Served through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.025966,
    "scale_divisor": 5
  },
  {
    "model": "mistral-small-3-2",
    "bench": "cptub-speakleash-avg",
    "score": 0.765489,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Small-3.2-24B-Instruct-2506  (API FP8)'; Space Average = mean(Sentiment 4.006, Language understanding 4.005, Phraseology 3.995, Tricky questions 3.303) = 3.8274 on a 0-5 judge scale, divided by 5. Served FP8 through an API. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.827445,
    "scale_divisor": 5
  },
  {
    "model": "gpt-oss-120b",
    "bench": "cptub-speakleash-avg",
    "score": 0.764497,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'openai/gpt-oss-120b  (API)'; Space Average = mean(Sentiment 3.942, Language understanding 3.970, Phraseology 3.490, Tricky questions 3.888) = 3.8225 on a 0-5 judge scale, divided by 5. Served through an API; reasoning effort not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.822487,
    "scale_divisor": 5
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.649411,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16'; Space Average = mean(Sentiment 3.526, Language understanding 3.270, Phraseology 2.760, Tricky questions 3.433) = 3.2471 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.247056,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-8b",
    "bench": "cptub-speakleash-avg",
    "score": 0.612382,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3-8B non-thinking (API)'; Space Average = mean(Sentiment 3.494, Language understanding 3.225, Phraseology 2.765, Tricky questions 2.764) = 3.0619 on a 0-5 judge scale, divided by 5. Hybrid model run with thinking disabled (only published setting). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.061909,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-14b",
    "bench": "cptub-speakleash-avg",
    "score": 0.702336,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3-14B non-thinking (API)'; Space Average = mean(Sentiment 3.910, Language understanding 3.560, Phraseology 3.245, Tricky questions 3.331) = 3.5117 on a 0-5 judge scale, divided by 5. Hybrid model run with thinking disabled (only published setting). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.511679,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-32b",
    "bench": "cptub-speakleash-avg",
    "score": 0.742071,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3-32B non-thinking (API)'; Space Average = mean(Sentiment 4.135, Language understanding 3.910, Phraseology 3.235, Tricky questions 3.562) = 3.7104 on a 0-5 judge scale, divided by 5. Hybrid model run with thinking disabled (only published setting). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.710353,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "cptub-speakleash-avg",
    "score": 0.782187,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3-235B-A22B non-thinking (API)'; Space Average = mean(Sentiment 4.167, Language understanding 4.155, Phraseology 3.485, Tricky questions 3.837) = 3.9109 on a 0-5 judge scale, divided by 5. April 2025 hybrid checkpoint run with thinking disabled (only published setting). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.910936,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-5-9b",
    "bench": "cptub-speakleash-avg",
    "score": 0.655163,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3.5-9B non-thinking (API, FP8)'; Space Average = mean(Sentiment 3.013, Language understanding 2.975, Phraseology 3.475, Tricky questions 3.640) = 3.2758 on a 0-5 judge scale, divided by 5. Thinking disabled, served FP8 (only published setting). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.275817,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-5-27b",
    "bench": "cptub-speakleash-avg",
    "score": 0.867272,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3.5-27B thinking (API)'; Space Average = mean(Sentiment 4.423, Language understanding 4.205, Phraseology 4.105, Tricky questions 4.612) = 4.3364 on a 0-5 judge scale, divided by 5. Thinking mode (the non-thinking row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.336359,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.844741,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3.5-35B-A3B thinking (API)'; Space Average = mean(Sentiment 4.103, Language understanding 3.940, Phraseology 4.150, Tricky questions 4.702) = 4.2237 on a 0-5 judge scale, divided by 5. Thinking mode (the non-thinking row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.223703,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-6-27b",
    "bench": "cptub-speakleash-avg",
    "score": 0.882439,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3.6-27B reasoning (API, FP8)'; Space Average = mean(Sentiment 4.583, Language understanding 4.215, Phraseology 4.210, Tricky questions 4.640) = 4.4122 on a 0-5 judge scale, divided by 5. Reasoning mode, served FP8 (the non-reasoning row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.412196,
    "scale_divisor": 5
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "cptub-speakleash-avg",
    "score": 0.872685,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3.5-122B-A10B reasoning (API, FP8)'; Space Average = mean(Sentiment 4.359, Language understanding 4.180, Phraseology 4.190, Tricky questions 4.725) = 4.3634 on a 0-5 judge scale, divided by 5. Reasoning mode, served FP8 (the non-reasoning row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.363423,
    "scale_divisor": 5
  },
  {
    "model": "gemma-4-31b",
    "bench": "cptub-speakleash-avg",
    "score": 0.862259,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'google/gemma-4-31B-it reasoning (API, FP8)'; Space Average = mean(Sentiment 4.583, Language understanding 4.380, Phraseology 3.765, Tricky questions 4.517) = 4.3113 on a 0-5 judge scale, divided by 5. Reasoning mode, served FP8 (the non-reasoning row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.311297,
    "scale_divisor": 5
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "cptub-speakleash-avg",
    "score": 0.842647,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'google/gemma-4-26B-A4B-it reasoning (API, FP8)'; Space Average = mean(Sentiment 4.391, Language understanding 4.080, Phraseology 3.910, Tricky questions 4.472) = 4.2132 on a 0-5 judge scale, divided by 5. Reasoning mode, served FP8 (the non-reasoning row is not imported). No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.213234,
    "scale_divisor": 5
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "cptub-speakleash-avg",
    "score": 0.883335,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-models/Muse-Glimmer-30B reasoning (API, FP8)'; Space Average = mean(Sentiment 4.519, Language understanding 4.395, Phraseology 4.230, Tricky questions 4.522) = 4.4167 on a 0-5 judge scale, divided by 5. Reasoning mode, served FP8. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.416676,
    "scale_divisor": 5
  },
  {
    "model": "gpt-5",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.715,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 71.5% +/- 1.8 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.694,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 69.4% +/- 3.0 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-mini",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.71,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-mini/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 71% +/- 1.8 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-mini",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.606,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-mini/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 60.6% +/- 2.7 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-mini",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.477,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-mini/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 47.7% +/- 3.3 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-nano",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.648,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-nano/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-nano' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 64.8% +/- 1.9 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-nano",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.406,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-nano/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-nano' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 40.6% +/- 2.7 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-5-nano",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.034,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-5-nano/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-5-nano' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 3.4% +/- 1.2 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.664,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-120b/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-120b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 66.4% +/- 1.9 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.509,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-120b/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-120b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 50.9% +/- 2.8 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.115,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-120b/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-120b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 11.5% +/- 2.1 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.64,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-20b/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-20b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 64% +/- 1.9 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.385,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-20b/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-20b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 38.5% +/- 2.7 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.026,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-oss-20b/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-oss-20b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 2.6% +/- 1.0 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Reasoning effort not stated. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.646,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-405b/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-11-29",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-405b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 64.6% +/- 1.9 (divided by 100); study 4d748972-6d35-4489-a197-138b656a7db3; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.433,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-405b/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-405b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 43.3% +/- 2.7 (divided by 100); study 2024-10-25_17-34-45; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.072,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-405b/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-11-29",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-405b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 7.2% +/- 1.7 (divided by 100); study 528da1f2-1949-41dc-b988-85f19f435af2; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-405b",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.24,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-405b/webarena.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-12-01",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-405b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 24.0% +/- 1.5 (divided by 100); study aaeca13d-0cf5-444f-8445-590350b54746; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.576,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-70b/miniwob.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-70b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 57.6% +/- 2.0 (divided by 100); study 2024-10-25_06-08-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.279,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-70b/workarena-l1.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-70b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 27.9% +/- 2.5 (divided by 100); study 2024-10-23_14-17-40; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.021,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-70b/workarena-l2.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-10-24",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-70b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 2.1% +/- 0.9 (divided by 100); study 2024-10-23_17-10-46; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.184,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Llama-3.1-70b/webarena.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2024-12-02",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Llama-3.1-70b' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 18.4% +/- 1.4 (divided by 100); study fc5747bc-d998-4942-a0eb-e55a3ccc1cb3; benchmark_tuned=No, followed_evaluation_protocol=Yes. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.307,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4.1-Mini/webarena.json",
    "source_id": "servicenow-browsergym-leaderboard",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4.1-Mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 30.7% +/- 2.4 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Agent scaffold result, not a bare-model score."
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "open-pt-llm-avg9",
    "score": 0.8558,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'Gemini 2.5 Flash [2025-09-01]' (source=external, precision=?); published average 85.58 (divided by 100). API run (external), September 2025 (after GA); thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 85.58,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 90.97,
      "bluex": 86.51,
      "oab_exams": 83.55,
      "assin2_rte": 93.86,
      "assin2_sts": 87.15,
      "faquad_nli": 85.79,
      "hatebr_offensive": 89.33,
      "portuguese_hate_speech": 75.03,
      "tweetsentbr": 78.01
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "open-pt-llm-avg9",
    "score": 0.8386,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'GPT 5 Mini [reasoning] (2025-08-07) [2025-09-01]' (source=external, precision=?); published average 83.86 (divided by 100). API run (external), reasoning enabled; effort not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 83.86,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 95.66,
      "bluex": 91.38,
      "oab_exams": 71.85,
      "assin2_rte": 94.87,
      "assin2_sts": 81.52,
      "faquad_nli": 79.6,
      "hatebr_offensive": 93.06,
      "portuguese_hate_speech": 74.77,
      "tweetsentbr": 72.08
    }
  },
  {
    "model": "gpt-5",
    "bench": "open-pt-llm-avg9",
    "score": 0.8342,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'GPT 5 [reasoning: minimal] (2025-08-07) [2025-09-01]' (source=external, precision=?); published average 83.42 (divided by 100). API run (external) with reasoning effort minimal, the only GPT-5 setting published. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 83.42,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 84.32,
      "bluex": 78.86,
      "oab_exams": 81.05,
      "assin2_rte": 94.98,
      "assin2_sts": 74.98,
      "faquad_nli": 90.49,
      "hatebr_offensive": 92.33,
      "portuguese_hate_speech": 75.02,
      "tweetsentbr": 78.78
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.8256,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-V3.1 (API) [2025-09-01]' (source=external, precision=?); published average 82.56 (divided by 100). API run (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 82.56,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 88.87,
      "bluex": 81.78,
      "oab_exams": 70.39,
      "assin2_rte": 94.93,
      "assin2_sts": 80.82,
      "faquad_nli": 84.07,
      "hatebr_offensive": 92.12,
      "portuguese_hate_speech": 74.23,
      "tweetsentbr": 75.84
    }
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "open-pt-llm-avg9",
    "score": 0.8233,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'Gemini 2.5 Flash Lite [reasoning: low] [2025-09-01]' (source=external, precision=?); published average 82.33 (divided by 100). API run (external), reasoning low (highest of the two published settings; no-reasoning row not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 82.33,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 90.13,
      "bluex": 84.01,
      "oab_exams": 69.43,
      "assin2_rte": 94.65,
      "assin2_sts": 75.56,
      "faquad_nli": 87.04,
      "hatebr_offensive": 90.81,
      "portuguese_hate_speech": 74.16,
      "tweetsentbr": 75.2
    }
  },
  {
    "model": "llama-3-1-405b",
    "bench": "open-pt-llm-avg9",
    "score": 0.8209,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.1-405B-Instruct (Vertex AI) [2024-08-20]' (source=external, precision=?); published average 82.09 (divided by 100). API run on Vertex AI (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 82.09,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 85.23,
      "bluex": 80.11,
      "oab_exams": 76.4,
      "assin2_rte": 94.76,
      "assin2_sts": 78.88,
      "faquad_nli": 82.51,
      "hatebr_offensive": 90.74,
      "portuguese_hate_speech": 71.91,
      "tweetsentbr": 78.21
    }
  },
  {
    "model": "kimi-k2",
    "bench": "open-pt-llm-avg9",
    "score": 0.8185,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'moonshotai/Kimi-K2-Instruct (API) [2025-09-01]' (source=external, precision=?); published average 81.85 (divided by 100). API run (external) labelled with the original Kimi-K2-Instruct id (0711), evaluated before the 0905 release. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 81.85,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 87.89,
      "bluex": 82.75,
      "oab_exams": 69.7,
      "assin2_rte": 94.36,
      "assin2_sts": 77.6,
      "faquad_nli": 85.31,
      "hatebr_offensive": 89.42,
      "portuguese_hate_speech": 75.36,
      "tweetsentbr": 74.28
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "open-pt-llm-avg9",
    "score": 0.8151,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-V3-0324 (API) [2025-04-03]' (source=external, precision=?); published average 81.51 (divided by 100). API run (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 81.51,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 89.01,
      "bluex": 84.14,
      "oab_exams": 71.48,
      "assin2_rte": 94.22,
      "assin2_sts": 81.46,
      "faquad_nli": 79.68,
      "hatebr_offensive": 90.6,
      "portuguese_hate_speech": 72.62,
      "tweetsentbr": 70.37
    }
  },
  {
    "model": "qwen2-5-72b",
    "bench": "open-pt-llm-avg9",
    "score": 0.8123,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-09-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-72B-Instruct [bfloat16]' (source=hub, precision=bfloat16); published average 81.23 (divided by 100). Local harness run, bfloat16 (the 4-bit and API rows are not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 81.23,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 84.74,
      "bluex": 79,
      "oab_exams": 67.43,
      "assin2_rte": 95.22,
      "assin2_sts": 82.08,
      "faquad_nli": 81.3,
      "hatebr_offensive": 87.95,
      "portuguese_hate_speech": 77.35,
      "tweetsentbr": 76
    }
  },
  {
    "model": "qwen2-5-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.8022,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-09-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-32B-Instruct' (source=hub, precision=bfloat16); published average 80.22 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 80.22,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 84.81,
      "bluex": 78.3,
      "oab_exams": 63.19,
      "assin2_rte": 94.73,
      "assin2_sts": 78.88,
      "faquad_nli": 82.27,
      "hatebr_offensive": 91.92,
      "portuguese_hate_speech": 73.26,
      "tweetsentbr": 74.63
    }
  },
  {
    "model": "llama-4-maverick",
    "bench": "open-pt-llm-avg9",
    "score": 0.8013,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-4-Maverick-17B-128E-Instruct (FireworksAI API) [2025-04-05]' (source=external, precision=?); published average 80.13 (divided by 100). API run on Fireworks AI (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 80.13,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 87.75,
      "bluex": 81.22,
      "oab_exams": 72.85,
      "assin2_rte": 93.29,
      "assin2_sts": 73.33,
      "faquad_nli": 78.24,
      "hatebr_offensive": 90.48,
      "portuguese_hate_speech": 72.31,
      "tweetsentbr": 71.65
    }
  },
  {
    "model": "qwen3-14b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7986,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-14B' (source=hub, precision=bfloat16); published average 79.86 (divided by 100). Local harness run, bfloat16; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.86,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 90.34,
      "bluex": 83.31,
      "oab_exams": 56.95,
      "assin2_rte": 93.33,
      "assin2_sts": 76.03,
      "faquad_nli": 85.82,
      "hatebr_offensive": 86.88,
      "portuguese_hate_speech": 72.34,
      "tweetsentbr": 73.7
    }
  },
  {
    "model": "gpt-5-nano",
    "bench": "open-pt-llm-avg9",
    "score": 0.7983,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-09-01",
    "note": "Open PT-LLM Leaderboard row 'GPT 5 Nano [reasoning] (2025-08-07) [2025-09-01]' (source=external, precision=?); published average 79.83 (divided by 100). API run (external), reasoning enabled; effort not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.83,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 90.13,
      "bluex": 85.26,
      "oab_exams": 59.13,
      "assin2_rte": 94.93,
      "assin2_sts": 71.58,
      "faquad_nli": 80.25,
      "hatebr_offensive": 91.7,
      "portuguese_hate_speech": 71.67,
      "tweetsentbr": 73.86
    }
  },
  {
    "model": "llama-3-1-70b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7934,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3.1-70B-Instruct [bfloat16]' (source=hub, precision=bfloat16); published average 79.34 (divided by 100). Local harness run, bfloat16 (4-bit row not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.34,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 82.09,
      "bluex": 75.1,
      "oab_exams": 67.15,
      "assin2_rte": 94.44,
      "assin2_sts": 75.17,
      "faquad_nli": 86.2,
      "hatebr_offensive": 89.87,
      "portuguese_hate_speech": 70.69,
      "tweetsentbr": 73.37
    }
  },
  {
    "model": "llama-3-3-70b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7918,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.3-70B-Instruct (Vertex AI) [2025-04-03]' (source=external, precision=?); published average 79.18 (divided by 100). API run on Vertex AI (external); the hub row is 4-bit and not imported. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.18,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 83.21,
      "bluex": 75.94,
      "oab_exams": 67.33,
      "assin2_rte": 94.07,
      "assin2_sts": 72.76,
      "faquad_nli": 87.88,
      "hatebr_offensive": 90.24,
      "portuguese_hate_speech": 70.42,
      "tweetsentbr": 70.77
    }
  },
  {
    "model": "qwen2-5-14b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7902,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-09-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-14B-Instruct' (source=hub, precision=bfloat16); published average 79.02 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.02,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 80.76,
      "bluex": 73.3,
      "oab_exams": 61.5,
      "assin2_rte": 94.69,
      "assin2_sts": 84.45,
      "faquad_nli": 79.63,
      "hatebr_offensive": 89.02,
      "portuguese_hate_speech": 74.91,
      "tweetsentbr": 72.89
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "open-pt-llm-avg9",
    "score": 0.7888,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-30B-A3B-Instruct-2507' (source=hub, precision=bfloat16); published average 78.88 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 78.88,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 83.28,
      "bluex": 76.91,
      "oab_exams": 60.87,
      "assin2_rte": 94.57,
      "assin2_sts": 81.06,
      "faquad_nli": 76.79,
      "hatebr_offensive": 89.17,
      "portuguese_hate_speech": 72.49,
      "tweetsentbr": 74.79
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7871,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-08",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3-27b-it (GoogleAI API) [2025-04-08]' (source=external, precision=?); published average 78.71 (divided by 100). API run on Google AI (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 78.71,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 81.46,
      "bluex": 73.85,
      "oab_exams": 61.59,
      "assin2_rte": 94.11,
      "assin2_sts": 81.48,
      "faquad_nli": 81.43,
      "hatebr_offensive": 87.29,
      "portuguese_hate_speech": 72.65,
      "tweetsentbr": 74.49
    }
  },
  {
    "model": "llama-4-scout",
    "bench": "open-pt-llm-avg9",
    "score": 0.7868,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-4-Scout-17B-16E-Instruct (Groq API) [2025-04-05]' (source=external, precision=?); published average 78.68 (divided by 100). API run on Groq (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 78.68,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 80.55,
      "bluex": 72.18,
      "oab_exams": 68.15,
      "assin2_rte": 93.13,
      "assin2_sts": 77.42,
      "faquad_nli": 85.67,
      "hatebr_offensive": 88.14,
      "portuguese_hate_speech": 70.09,
      "tweetsentbr": 72.77
    }
  },
  {
    "model": "qwen3-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7835,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-8B' (source=hub, precision=bfloat16); published average 78.35 (divided by 100). Local harness run, bfloat16; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 78.35,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 87.47,
      "bluex": 80.67,
      "oab_exams": 52.89,
      "assin2_rte": 92.59,
      "assin2_sts": 75.53,
      "faquad_nli": 83.26,
      "hatebr_offensive": 86.53,
      "portuguese_hate_speech": 74.84,
      "tweetsentbr": 71.35
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7639,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-04",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3-12b-it' (source=hub, precision=bfloat16); published average 76.39 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.39,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 76.28,
      "bluex": 66.62,
      "oab_exams": 56.63,
      "assin2_rte": 93.86,
      "assin2_sts": 83.22,
      "faquad_nli": 76.61,
      "hatebr_offensive": 86.2,
      "portuguese_hate_speech": 74.24,
      "tweetsentbr": 73.86
    }
  },
  {
    "model": "phi-4",
    "bench": "open-pt-llm-avg9",
    "score": 0.761,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'microsoft/phi-4' (source=hub, precision=bfloat16); published average 76.1 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.1,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 79.99,
      "bluex": 70.24,
      "oab_exams": 60.77,
      "assin2_rte": 93.26,
      "assin2_sts": 84.54,
      "faquad_nli": 73.2,
      "hatebr_offensive": 81.51,
      "portuguese_hate_speech": 68.6,
      "tweetsentbr": 72.83
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "open-pt-llm-avg9",
    "score": 0.7469,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-4B-Instruct-2507' (source=hub, precision=bfloat16); published average 74.69 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 74.69,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 75.09,
      "bluex": 67.73,
      "oab_exams": 51.98,
      "assin2_rte": 93.79,
      "assin2_sts": 75.08,
      "faquad_nli": 83.1,
      "hatebr_offensive": 88.98,
      "portuguese_hate_speech": 67.51,
      "tweetsentbr": 68.91
    }
  },
  {
    "model": "qwen2-5-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7384,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-09-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-7B-Instruct' (source=hub, precision=bfloat16); published average 73.84 (divided by 100). Local harness run, bfloat16, main revision (a second row pinned to revision bb46c15 is not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 73.84,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 75.02,
      "bluex": 64.12,
      "oab_exams": 53.39,
      "assin2_rte": 93.83,
      "assin2_sts": 76.11,
      "faquad_nli": 82.81,
      "hatebr_offensive": 77.98,
      "portuguese_hate_speech": 71.63,
      "tweetsentbr": 69.66
    }
  },
  {
    "model": "llama-3-1-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7021,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3.1-8B-Instruct' (source=hub, precision=bfloat16); published average 70.21 (divided by 100). Local harness run, bfloat16 (a float16 row under the renamed repo meta-llama/Llama-3.1-8B-Instruct, 71.24, is not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 70.21,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 71.03,
      "bluex": 58.83,
      "oab_exams": 51.66,
      "assin2_rte": 92.27,
      "assin2_sts": 74.6,
      "faquad_nli": 72.63,
      "hatebr_offensive": 87.2,
      "portuguese_hate_speech": 62.09,
      "tweetsentbr": 61.61
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6811,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-1.7B' (source=hub, precision=bfloat16); published average 68.11 (divided by 100). Local harness run, bfloat16; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 68.11,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 69.98,
      "bluex": 64.26,
      "oab_exams": 37.04,
      "assin2_rte": 91.04,
      "assin2_sts": 57.92,
      "faquad_nli": 77.77,
      "hatebr_offensive": 82.93,
      "portuguese_hate_speech": 68.57,
      "tweetsentbr": 63.51
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6754,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-07-08",
    "note": "Open PT-LLM Leaderboard row 'HuggingFaceTB/SmolLM3-3B' (source=hub, precision=bfloat16); published average 67.54 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 67.54,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 61.02,
      "bluex": 48.54,
      "oab_exams": 42,
      "assin2_rte": 92.77,
      "assin2_sts": 76.52,
      "faquad_nli": 72.77,
      "hatebr_offensive": 78.73,
      "portuguese_hate_speech": 69.58,
      "tweetsentbr": 65.95
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "open-pt-llm-avg9",
    "score": 0.673,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-04",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3-4b-it' (source=hub, precision=bfloat16); published average 67.3 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 67.3,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 63.47,
      "bluex": 52.29,
      "oab_exams": 43.87,
      "assin2_rte": 90.43,
      "assin2_sts": 71.93,
      "faquad_nli": 69.42,
      "hatebr_offensive": 83.91,
      "portuguese_hate_speech": 63.88,
      "tweetsentbr": 66.48
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "open-pt-llm-avg9",
    "score": 0.6647,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-4-mini-instruct' (source=hub, precision=bfloat16); published average 66.47 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 66.47,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 59.27,
      "bluex": 49.37,
      "oab_exams": 41.64,
      "assin2_rte": 90.4,
      "assin2_sts": 70.19,
      "faquad_nli": 72.89,
      "hatebr_offensive": 84.29,
      "portuguese_hate_speech": 68.67,
      "tweetsentbr": 61.51
    }
  },
  {
    "model": "llama-3-2-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6479,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.2-3B-Instruct' (source=hub, precision=bfloat16); published average 64.79 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 64.79,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 59.06,
      "bluex": 52.29,
      "oab_exams": 44.92,
      "assin2_rte": 85.04,
      "assin2_sts": 61.89,
      "faquad_nli": 66.97,
      "hatebr_offensive": 86.31,
      "portuguese_hate_speech": 67.51,
      "tweetsentbr": 59.1
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5634,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-07-15",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3n-E2B-it' (source=hub, precision=bfloat16); published average 56.34 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 56.34,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 59.62,
      "bluex": 47.29,
      "oab_exams": 42.64,
      "assin2_rte": 56.42,
      "assin2_sts": 69.31,
      "faquad_nli": 43.97,
      "hatebr_offensive": 56.95,
      "portuguese_hate_speech": 66.61,
      "tweetsentbr": 64.22
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5418,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-04",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3-1b-it' (source=hub, precision=bfloat16); published average 54.18 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 54.18,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 38.77,
      "bluex": 32.96,
      "oab_exams": 31.57,
      "assin2_rte": 81.67,
      "assin2_sts": 44.18,
      "faquad_nli": 57.63,
      "hatebr_offensive": 82.64,
      "portuguese_hate_speech": 58.4,
      "tweetsentbr": 59.77
    }
  },
  {
    "model": "qwen3-0-6b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5028,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-0.6B' (source=hub, precision=bfloat16); published average 50.28 (divided by 100). Local harness run, bfloat16; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 50.28,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 46.47,
      "bluex": 43.39,
      "oab_exams": 32.98,
      "assin2_rte": 79,
      "assin2_sts": 15.25,
      "faquad_nli": 55.35,
      "hatebr_offensive": 68.09,
      "portuguese_hate_speech": 59.66,
      "tweetsentbr": 52.34
    }
  },
  {
    "model": "llama-3-2-1b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3587,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.2-1B-Instruct' (source=hub, precision=bfloat16); published average 35.87 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 35.87,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 34.78,
      "bluex": 30.6,
      "oab_exams": 31.12,
      "assin2_rte": 36.89,
      "assin2_sts": 17.88,
      "faquad_nli": 51.38,
      "hatebr_offensive": 34.28,
      "portuguese_hate_speech": 41.23,
      "tweetsentbr": 44.68
    }
  },
  {
    "model": "gemma-3-270m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2819,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-3-270m-it [bfloat16]' (source=hub, precision=bfloat16); published average 28.19 (divided by 100). Local harness run, bfloat16 (a float16 row scoring 0.0 is not imported). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 28.19,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 20.99,
      "bluex": 18.5,
      "oab_exams": 24.6,
      "assin2_rte": 33.42,
      "assin2_sts": 21.58,
      "faquad_nli": 43.97,
      "hatebr_offensive": 33.43,
      "portuguese_hate_speech": 41.65,
      "tweetsentbr": 15.6
    }
  },
  {
    "model": "gemma-4-31b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.785457,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-4-31B-it' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 78.5457 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 78.545714,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 82.14,
      "GSM8K": 80.1,
      "DC-Homograph": 95.37,
      "MC-Homograph": 96.54,
      "PiQA": 94.79,
      "Proverb-Quiz": 90.27,
      "VerbEval": 84.33,
      "Winogrande": 89.02,
      "Arc-Challenge": 95.51,
      "Arc-Easy": 97.33,
      "Feqh": 56.57,
      "Hallucination (Truthfulness)": 74.37,
      "P-Hellaswag": 88.54,
      "Law": 56,
      "AUT Multiple Choice": 58.5,
      "Parsi Literature": 49.29,
      "BoolQA": 96.2,
      "Reading Comprehension": 62.34,
      "PartExpert": 62.25,
      "MMLU Pro": 62.2,
      "Iranian Social Norms": 77.8
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.760476,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3.8-27B' (query id Qwen/Qwen3.8-27B, precision unknown); Average Accuracy = mean of the 21 published task scores = 76.0476 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 76.047619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 76.16,
      "GSM8K": 79.08,
      "DC-Homograph": 83.3,
      "MC-Homograph": 98.15,
      "PiQA": 96.77,
      "Proverb-Quiz": 94.99,
      "VerbEval": 81.35,
      "Winogrande": 76.09,
      "Arc-Challenge": 90.52,
      "Arc-Easy": 96.47,
      "Feqh": 98.18,
      "Hallucination (Truthfulness)": 34.86,
      "P-Hellaswag": 65.19,
      "Law": 85.82,
      "AUT Multiple Choice": 36.33,
      "Parsi Literature": 59.5,
      "BoolQA": 44.92,
      "Reading Comprehension": 96.1,
      "PartExpert": 62.3,
      "MMLU Pro": 67.12,
      "Iranian Social Norms": 73.8
    }
  },
  {
    "model": "gemma-4-e4b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.590362,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-4-E4B-it' (query id google/gemma-4-E4B-it, precision unknown); Average Accuracy = mean of the 21 published task scores = 59.0362 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 59.03619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 59.72,
      "GSM8K": 59.18,
      "DC-Homograph": 16,
      "MC-Homograph": 62.04,
      "PiQA": 88.48,
      "Proverb-Quiz": 86.99,
      "VerbEval": 68.11,
      "Winogrande": 68.84,
      "Arc-Challenge": 69.97,
      "Arc-Easy": 84.29,
      "Feqh": 91.12,
      "Hallucination (Truthfulness)": 29.14,
      "P-Hellaswag": 23.64,
      "Law": 81.41,
      "AUT Multiple Choice": 35,
      "Parsi Literature": 42.8,
      "BoolQA": 37.19,
      "Reading Comprehension": 91,
      "PartExpert": 66.1,
      "MMLU Pro": 43.34,
      "Iranian Social Norms": 35.4
    }
  },
  {
    "model": "gemma-4-e2b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.504819,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-4-E2B-it' (query id google/gemma-4-E2B-it, precision unknown); Average Accuracy = mean of the 21 published task scores = 50.4819 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 50.481905,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 50.82,
      "GSM8K": 42.86,
      "DC-Homograph": 12.8,
      "MC-Homograph": 49.07,
      "PiQA": 76.96,
      "Proverb-Quiz": 70.37,
      "VerbEval": 47.03,
      "Winogrande": 46.13,
      "Arc-Challenge": 57.48,
      "Arc-Easy": 65.06,
      "Feqh": 76.79,
      "Hallucination (Truthfulness)": 27.43,
      "P-Hellaswag": 53.79,
      "Law": 73.62,
      "AUT Multiple Choice": 32.67,
      "Parsi Literature": 37.3,
      "BoolQA": 31.79,
      "Reading Comprehension": 88,
      "PartExpert": 58,
      "MMLU Pro": 34.85,
      "Iranian Social Norms": 27.3
    }
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.706843,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-4-26B-A4B-it' (query id google/gemma-4-26B-A4B-it, precision unknown); Average Accuracy = mean of the 21 published task scores = 70.6843 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 70.684286,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 70.91,
      "GSM8K": 74.74,
      "DC-Homograph": 64,
      "MC-Homograph": 78.7,
      "PiQA": 94.01,
      "Proverb-Quiz": 91.49,
      "VerbEval": 78.38,
      "Winogrande": 78.05,
      "Arc-Challenge": 76.26,
      "Arc-Easy": 93.06,
      "Feqh": 95.94,
      "Hallucination (Truthfulness)": 38.29,
      "P-Hellaswag": 58.33,
      "Law": 85.6,
      "AUT Multiple Choice": 44,
      "Parsi Literature": 53.2,
      "BoolQA": 44.4,
      "Reading Comprehension": 96.1,
      "PartExpert": 63.06,
      "MMLU Pro": 54.55,
      "Iranian Social Norms": 51.3
    }
  },
  {
    "model": "gpt-4-1",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.743805,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4.1-2025-04-14' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 74.3805 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 74.380476,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 90.82,
      "GSM8K": 25.3,
      "DC-Homograph": 89.81,
      "MC-Homograph": 95.39,
      "PiQA": 95.9,
      "Proverb-Quiz": 95.14,
      "VerbEval": 83.04,
      "Winogrande": 85.92,
      "Arc-Challenge": 95.3,
      "Arc-Easy": 96.68,
      "Feqh": 52,
      "Hallucination (Truthfulness)": 77.43,
      "P-Hellaswag": 85.67,
      "Law": 53.67,
      "AUT Multiple Choice": 66.6,
      "Parsi Literature": 45.82,
      "BoolQA": 94.7,
      "Reading Comprehension": 44.82,
      "PartExpert": 59.92,
      "MMLU Pro": 50.5,
      "Iranian Social Norms": 77.56
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.724471,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'deepseek-v3-03-24' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 72.4471 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 72.447143,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 85.71,
      "GSM8K": 53.1,
      "DC-Homograph": 83.33,
      "MC-Homograph": 94.24,
      "PiQA": 91.39,
      "Proverb-Quiz": 84.86,
      "VerbEval": 81.11,
      "Winogrande": 76.71,
      "Arc-Challenge": 92.31,
      "Arc-Easy": 96.58,
      "Feqh": 42.29,
      "Hallucination (Truthfulness)": 55.54,
      "P-Hellaswag": 85.3,
      "Law": 46,
      "AUT Multiple Choice": 65.6,
      "Parsi Literature": 44.66,
      "BoolQA": 95.3,
      "Reading Comprehension": 63.79,
      "PartExpert": 58.46,
      "MMLU Pro": 53.4,
      "Iranian Social Norms": 71.71
    }
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.700576,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4.1-mini-2025-04-14' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 70.0576 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 70.057619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 79.34,
      "GSM8K": 60.3,
      "DC-Homograph": 66.67,
      "MC-Homograph": 94.24,
      "PiQA": 92.69,
      "Proverb-Quiz": 82.97,
      "VerbEval": 77.99,
      "Winogrande": 80.07,
      "Arc-Challenge": 91.88,
      "Arc-Easy": 96.15,
      "Feqh": 37.71,
      "Hallucination (Truthfulness)": 66.55,
      "P-Hellaswag": 84.57,
      "Law": 44.33,
      "AUT Multiple Choice": 53.5,
      "Parsi Literature": 41.18,
      "BoolQA": 93.7,
      "Reading Comprehension": 51.85,
      "PartExpert": 54.37,
      "MMLU Pro": 47.8,
      "Iranian Social Norms": 73.35
    }
  },
  {
    "model": "command-a",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.700176,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-a-03-2025' (query id CohereLabs/c4ai-command-a-03-2025, precision BF16); Average Accuracy = mean of the 21 published task scores = 70.0176 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 70.017619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 79.34,
      "GSM8K": 35.6,
      "DC-Homograph": 75.93,
      "MC-Homograph": 94.93,
      "PiQA": 93.99,
      "Proverb-Quiz": 87.3,
      "VerbEval": 70.65,
      "Winogrande": 80.34,
      "Arc-Challenge": 91.03,
      "Arc-Easy": 96.9,
      "Feqh": 46.86,
      "Hallucination (Truthfulness)": 54.64,
      "P-Hellaswag": 84.28,
      "Law": 45,
      "AUT Multiple Choice": 61.3,
      "Parsi Literature": 41.57,
      "BoolQA": 92.6,
      "Reading Comprehension": 66.81,
      "PartExpert": 52.44,
      "MMLU Pro": 41.3,
      "Iranian Social Norms": 77.56
    }
  },
  {
    "model": "llama-4-scout",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.68999,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-4-Scout-17B-16E-Instruct' (query id meta-llama/Llama-4-Scout-17B-16E-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 68.9990 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 68.999048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 81.63,
      "GSM8K": 35.4,
      "DC-Homograph": 71.3,
      "MC-Homograph": 93.09,
      "PiQA": 90.29,
      "Proverb-Quiz": 83.78,
      "VerbEval": 79.25,
      "Winogrande": 78.3,
      "Arc-Challenge": 88.78,
      "Arc-Easy": 95.19,
      "Feqh": 41.14,
      "Hallucination (Truthfulness)": 53.18,
      "P-Hellaswag": 84.42,
      "Law": 38.67,
      "AUT Multiple Choice": 55.2,
      "Parsi Literature": 42.21,
      "BoolQA": 93.8,
      "Reading Comprehension": 66.62,
      "PartExpert": 54.55,
      "MMLU Pro": 47,
      "Iranian Social Norms": 75.18
    }
  },
  {
    "model": "llama-3-3-70b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.684686,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-3.3-70B-Instruct' (query id meta-llama/Llama-3.3-70B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 68.4686 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 68.468571,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 78.83,
      "GSM8K": 31.5,
      "DC-Homograph": 66.67,
      "MC-Homograph": 92.17,
      "PiQA": 90.29,
      "Proverb-Quiz": 78.92,
      "VerbEval": 68.32,
      "Winogrande": 81.93,
      "Arc-Challenge": 89.53,
      "Arc-Easy": 95.94,
      "Feqh": 39.43,
      "Hallucination (Truthfulness)": 66.68,
      "P-Hellaswag": 83.91,
      "Law": 45,
      "AUT Multiple Choice": 57.3,
      "Parsi Literature": 39.9,
      "BoolQA": 94.8,
      "Reading Comprehension": 64.57,
      "PartExpert": 52.62,
      "MMLU Pro": 42.7,
      "Iranian Social Norms": 76.83
    }
  },
  {
    "model": "llama-3-1-70b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.680348,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Meta-Llama-3.1-70B-Instruct' (query id meta-llama/Meta-Llama-3.1-70B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 68.0348 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 68.034762,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 78.83,
      "GSM8K": 29.3,
      "DC-Homograph": 63.89,
      "MC-Homograph": 92.17,
      "PiQA": 89.89,
      "Proverb-Quiz": 77.57,
      "VerbEval": 65.38,
      "Winogrande": 81.93,
      "Arc-Challenge": 89.74,
      "Arc-Easy": 95.61,
      "Feqh": 42.29,
      "Hallucination (Truthfulness)": 68.23,
      "P-Hellaswag": 85.67,
      "Law": 45,
      "AUT Multiple Choice": 57.2,
      "Parsi Literature": 38.61,
      "BoolQA": 95.2,
      "Reading Comprehension": 59.32,
      "PartExpert": 53.3,
      "MMLU Pro": 43.2,
      "Iranian Social Norms": 76.4
    }
  },
  {
    "model": "qwen2-5-72b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.655795,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-72B-Instruct' (query id Qwen/Qwen2.5-72B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 65.5795 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 65.579524,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 76.79,
      "GSM8K": 40.4,
      "DC-Homograph": 66.67,
      "MC-Homograph": 91.24,
      "PiQA": 88.59,
      "Proverb-Quiz": 68.92,
      "VerbEval": 62.17,
      "Winogrande": 78.03,
      "Arc-Challenge": 91.35,
      "Arc-Easy": 94.22,
      "Feqh": 34.29,
      "Hallucination (Truthfulness)": 63.25,
      "P-Hellaswag": 84.13,
      "Law": 39,
      "AUT Multiple Choice": 55.9,
      "Parsi Literature": 37.71,
      "BoolQA": 95.6,
      "Reading Comprehension": 43.67,
      "PartExpert": 53.03,
      "MMLU Pro": 39.1,
      "Iranian Social Norms": 73.11
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.646486,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3-27b-it' (query id google/gemma-3-27b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 64.6486 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 64.648571,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 73.72,
      "GSM8K": 28.3,
      "DC-Homograph": 63.89,
      "MC-Homograph": 92.4,
      "PiQA": 87.29,
      "Proverb-Quiz": 78.92,
      "VerbEval": 66.02,
      "Winogrande": 78.12,
      "Arc-Challenge": 88.35,
      "Arc-Easy": 94.22,
      "Feqh": 24.57,
      "Hallucination (Truthfulness)": 60.15,
      "P-Hellaswag": 83.39,
      "Law": 36.33,
      "AUT Multiple Choice": 55.2,
      "Parsi Literature": 40.93,
      "BoolQA": 91.4,
      "Reading Comprehension": 58.01,
      "PartExpert": 49.32,
      "MMLU Pro": 36.6,
      "Iranian Social Norms": 70.49
    }
  },
  {
    "model": "mistral-large-2",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.642552,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-Large-Instruct-2411' (query id mistralai/Mistral-Large-Instruct-2411, precision BF16); Average Accuracy = mean of the 21 published task scores = 64.2552 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 64.255238,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 71.94,
      "GSM8K": 34.6,
      "DC-Homograph": 75.93,
      "MC-Homograph": 94.01,
      "PiQA": 88.29,
      "Proverb-Quiz": 65.41,
      "VerbEval": 54.58,
      "Winogrande": 76.71,
      "Arc-Challenge": 88.35,
      "Arc-Easy": 94.33,
      "Feqh": 31.43,
      "Hallucination (Truthfulness)": 45.51,
      "P-Hellaswag": 83.84,
      "Law": 37.33,
      "AUT Multiple Choice": 55.1,
      "Parsi Literature": 37.19,
      "BoolQA": 87.5,
      "Reading Comprehension": 67.21,
      "PartExpert": 48.67,
      "MMLU Pro": 37.1,
      "Iranian Social Norms": 74.33
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.628067,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-32B' (query id Qwen/Qwen3-32B, precision BF16); Average Accuracy = mean of the 21 published task scores = 62.8067 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 62.806667,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 67.6,
      "GSM8K": 37.9,
      "DC-Homograph": 51.85,
      "MC-Homograph": 89.63,
      "PiQA": 87.69,
      "Proverb-Quiz": 64.59,
      "VerbEval": 56.35,
      "Winogrande": 71.48,
      "Arc-Challenge": 91.13,
      "Arc-Easy": 94.22,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 47.5,
      "P-Hellaswag": 83.47,
      "Law": 37,
      "AUT Multiple Choice": 48.3,
      "Parsi Literature": 39.12,
      "BoolQA": 91.1,
      "Reading Comprehension": 63.96,
      "PartExpert": 50.06,
      "MMLU Pro": 42.8,
      "Iranian Social Norms": 73.48
    }
  },
  {
    "model": "qwen2-5-32b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.627981,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-32B-Instruct' (query id Qwen/Qwen2.5-32B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 62.7981 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 62.798095,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 61.73,
      "GSM8K": 50.1,
      "DC-Homograph": 67.59,
      "MC-Homograph": 91.47,
      "PiQA": 83.98,
      "Proverb-Quiz": 63.24,
      "VerbEval": 54.58,
      "Winogrande": 80.07,
      "Arc-Challenge": 85.15,
      "Arc-Easy": 91.87,
      "Feqh": 38.86,
      "Hallucination (Truthfulness)": 59.22,
      "P-Hellaswag": 82.07,
      "Law": 42.33,
      "AUT Multiple Choice": 50.4,
      "Parsi Literature": 40.41,
      "BoolQA": 93.4,
      "Reading Comprehension": 28.11,
      "PartExpert": 46.78,
      "MMLU Pro": 37.4,
      "Iranian Social Norms": 70
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.617333,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3-12b-it' (query id google/gemma-3-12b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 61.7333 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 61.733333,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 68.37,
      "GSM8K": 20.2,
      "DC-Homograph": 67.59,
      "MC-Homograph": 91.24,
      "PiQA": 87.19,
      "Proverb-Quiz": 72.97,
      "VerbEval": 63.39,
      "Winogrande": 73.96,
      "Arc-Challenge": 83.33,
      "Arc-Easy": 93.26,
      "Feqh": 25.14,
      "Hallucination (Truthfulness)": 46.1,
      "P-Hellaswag": 83.17,
      "Law": 36.33,
      "AUT Multiple Choice": 49,
      "Parsi Literature": 40.03,
      "BoolQA": 87.6,
      "Reading Comprehension": 55.26,
      "PartExpert": 44.12,
      "MMLU Pro": 32.6,
      "Iranian Social Norms": 75.55
    }
  },
  {
    "model": "qwen3-14b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.577738,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-14B' (query id Qwen/Qwen3-14B, precision BF16); Average Accuracy = mean of the 21 published task scores = 57.7738 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 57.77381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 56.38,
      "GSM8K": 31.1,
      "DC-Homograph": 55.56,
      "MC-Homograph": 87.56,
      "PiQA": 77.18,
      "Proverb-Quiz": 53.78,
      "VerbEval": 54.36,
      "Winogrande": 67.32,
      "Arc-Challenge": 84.29,
      "Arc-Easy": 91.02,
      "Feqh": 29.14,
      "Hallucination (Truthfulness)": 44.54,
      "P-Hellaswag": 80.97,
      "Law": 34.67,
      "AUT Multiple Choice": 44.8,
      "Parsi Literature": 35.39,
      "BoolQA": 87.6,
      "Reading Comprehension": 44.36,
      "PartExpert": 43.22,
      "MMLU Pro": 35.5,
      "Iranian Social Norms": 74.51
    }
  },
  {
    "model": "qwen2-5-14b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.555724,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-14B-Instruct' (query id Qwen/Qwen2.5-14B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 55.5724 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 55.572381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 60.46,
      "GSM8K": 38.8,
      "DC-Homograph": 59.26,
      "MC-Homograph": 86.87,
      "PiQA": 76.98,
      "Proverb-Quiz": 54.59,
      "VerbEval": 51.21,
      "Winogrande": 51.21,
      "Arc-Challenge": 82.26,
      "Arc-Easy": 87.91,
      "Feqh": 32,
      "Hallucination (Truthfulness)": 26.99,
      "P-Hellaswag": 78.99,
      "Law": 34,
      "AUT Multiple Choice": 49.7,
      "Parsi Literature": 33.85,
      "BoolQA": 91.1,
      "Reading Comprehension": 22.46,
      "PartExpert": 43.41,
      "MMLU Pro": 34.6,
      "Iranian Social Norms": 70.37
    }
  },
  {
    "model": "qwen3-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.550214,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-8B' (query id Qwen/Qwen3-8B, precision BF16); Average Accuracy = mean of the 21 published task scores = 55.0214 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 55.021429,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 49.23,
      "GSM8K": 25.7,
      "DC-Homograph": 50.93,
      "MC-Homograph": 82.95,
      "PiQA": 75.98,
      "Proverb-Quiz": 51.89,
      "VerbEval": 47.93,
      "Winogrande": 61.91,
      "Arc-Challenge": 80.24,
      "Arc-Easy": 87.38,
      "Feqh": 28,
      "Hallucination (Truthfulness)": 38.46,
      "P-Hellaswag": 80.38,
      "Law": 29.67,
      "AUT Multiple Choice": 46,
      "Parsi Literature": 33.2,
      "BoolQA": 86.4,
      "Reading Comprehension": 66.38,
      "PartExpert": 38.31,
      "MMLU Pro": 31.1,
      "Iranian Social Norms": 63.41
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.549705,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3n-E4B-it' (query id google/gemma-3n-E4B-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 54.9705 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 54.970476,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 59.44,
      "GSM8K": 10,
      "DC-Homograph": 52.78,
      "MC-Homograph": 78.57,
      "PiQA": 82.28,
      "Proverb-Quiz": 65.95,
      "VerbEval": 50.87,
      "Winogrande": 65.46,
      "Arc-Challenge": 75.85,
      "Arc-Easy": 85.45,
      "Feqh": 28.57,
      "Hallucination (Truthfulness)": 44.93,
      "P-Hellaswag": 77.96,
      "Law": 33,
      "AUT Multiple Choice": 42.6,
      "Parsi Literature": 31.79,
      "BoolQA": 83.6,
      "Reading Comprehension": 46.92,
      "PartExpert": 38.8,
      "MMLU Pro": 27.3,
      "Iranian Social Norms": 72.26
    }
  },
  {
    "model": "qwen2-5-7b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.524757,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-7B-Instruct' (query id Qwen/Qwen2.5-7B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 52.4757 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 52.475714,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 51.02,
      "GSM8K": 18,
      "DC-Homograph": 52.78,
      "MC-Homograph": 79.26,
      "PiQA": 71.07,
      "Proverb-Quiz": 47.84,
      "VerbEval": 44.44,
      "Winogrande": 61.91,
      "Arc-Challenge": 72.33,
      "Arc-Easy": 81.5,
      "Feqh": 36.57,
      "Hallucination (Truthfulness)": 34.89,
      "P-Hellaswag": 74.8,
      "Law": 32.33,
      "AUT Multiple Choice": 42.6,
      "Parsi Literature": 31.27,
      "BoolQA": 82.5,
      "Reading Comprehension": 58.43,
      "PartExpert": 37.24,
      "MMLU Pro": 26.7,
      "Iranian Social Norms": 64.51
    }
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.520129,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3n-E2B-it' (query id google/gemma-3n-E2B-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 52.0129 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 52.012857,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 51.79,
      "GSM8K": 6.9,
      "DC-Homograph": 49.07,
      "MC-Homograph": 75.35,
      "PiQA": 78.78,
      "Proverb-Quiz": 59.73,
      "VerbEval": 47.69,
      "Winogrande": 62.36,
      "Arc-Challenge": 70.83,
      "Arc-Easy": 84.17,
      "Feqh": 21.14,
      "Hallucination (Truthfulness)": 45.6,
      "P-Hellaswag": 77.08,
      "Law": 32.33,
      "AUT Multiple Choice": 42,
      "Parsi Literature": 31.53,
      "BoolQA": 83.6,
      "Reading Comprehension": 41.88,
      "PartExpert": 36.04,
      "MMLU Pro": 23.3,
      "Iranian Social Norms": 71.1
    }
  },
  {
    "model": "llama-3-1-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.503076,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-3.1-8B-Instruct' (query id meta-llama/Llama-3.1-8B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 50.3076 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 50.307619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 52.55,
      "GSM8K": 12,
      "DC-Homograph": 43.52,
      "MC-Homograph": 79.03,
      "PiQA": 70.07,
      "Proverb-Quiz": 47.57,
      "VerbEval": 42.91,
      "Winogrande": 54.21,
      "Arc-Challenge": 68.91,
      "Arc-Easy": 80.11,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 6.76,
      "P-Hellaswag": 79.79,
      "Law": 32.67,
      "AUT Multiple Choice": 44.9,
      "Parsi Literature": 32.3,
      "BoolQA": 82.7,
      "Reading Comprehension": 62.45,
      "PartExpert": 37.62,
      "MMLU Pro": 25.7,
      "Iranian Social Norms": 70.98
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.490962,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3-4b-it' (query id google/gemma-3-4b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 49.0962 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 49.09619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 45.92,
      "GSM8K": 9.6,
      "DC-Homograph": 42.59,
      "MC-Homograph": 72.58,
      "PiQA": 72.77,
      "Proverb-Quiz": 53.78,
      "VerbEval": 45.3,
      "Winogrande": 55.09,
      "Arc-Challenge": 63.46,
      "Arc-Easy": 79.57,
      "Feqh": 21.14,
      "Hallucination (Truthfulness)": 46.04,
      "P-Hellaswag": 73.84,
      "Law": 27.67,
      "AUT Multiple Choice": 42.5,
      "Parsi Literature": 30.24,
      "BoolQA": 78.6,
      "Reading Comprehension": 47.28,
      "PartExpert": 34.7,
      "MMLU Pro": 22.8,
      "Iranian Social Norms": 65.55
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.413057,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-1.7B' (query id Qwen/Qwen3-1.7B, precision BF16); Average Accuracy = mean of the 21 published task scores = 41.3057 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 41.305714,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 32.91,
      "GSM8K": 7.9,
      "DC-Homograph": 48.15,
      "MC-Homograph": 63.13,
      "PiQA": 58.66,
      "Proverb-Quiz": 38.11,
      "VerbEval": 33.67,
      "Winogrande": 33.67,
      "Arc-Challenge": 52.78,
      "Arc-Easy": 64.17,
      "Feqh": 26.29,
      "Hallucination (Truthfulness)": 32.04,
      "P-Hellaswag": 69.14,
      "Law": 31,
      "AUT Multiple Choice": 32.6,
      "Parsi Literature": 29.99,
      "BoolQA": 72.1,
      "Reading Comprehension": 52.6,
      "PartExpert": 31.85,
      "MMLU Pro": 20.5,
      "Iranian Social Norms": 36.16
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.374848,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-3-1b-it' (query id google/gemma-3-1b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 37.4848 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 37.484762,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 26.02,
      "GSM8K": 4.3,
      "DC-Homograph": 49.07,
      "MC-Homograph": 51.15,
      "PiQA": 57.66,
      "Proverb-Quiz": 28.92,
      "VerbEval": 27.67,
      "Winogrande": 50.58,
      "Arc-Challenge": 36.43,
      "Arc-Easy": 46.1,
      "Feqh": 28,
      "Hallucination (Truthfulness)": 54.94,
      "P-Hellaswag": 63.92,
      "Law": 20.33,
      "AUT Multiple Choice": 29.1,
      "Parsi Literature": 24.97,
      "BoolQA": 63.9,
      "Reading Comprehension": 31.98,
      "PartExpert": 27.22,
      "MMLU Pro": 13.7,
      "Iranian Social Norms": 51.22
    }
  },
  {
    "model": "qwen3-0-6b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.362543,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-0.6B' (query id Qwen/Qwen3-0.6B, precision BF16); Average Accuracy = mean of the 21 published task scores = 36.2543 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 36.254286,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 24.49,
      "GSM8K": 4.4,
      "DC-Homograph": 53.7,
      "MC-Homograph": 46.77,
      "PiQA": 52.85,
      "Proverb-Quiz": 34.59,
      "VerbEval": 23.14,
      "Winogrande": 23.14,
      "Arc-Challenge": 31.73,
      "Arc-Easy": 37.22,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 54.16,
      "P-Hellaswag": 62.23,
      "Law": 26,
      "AUT Multiple Choice": 27.3,
      "Parsi Literature": 28.96,
      "BoolQA": 67.6,
      "Reading Comprehension": 38.1,
      "PartExpert": 27.6,
      "MMLU Pro": 17.1,
      "Iranian Social Norms": 50.55
    }
  },
  {
    "model": "llama-3-2-1b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.349924,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-3.2-1B-Instruct' (query id meta-llama/Llama-3.2-1B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 34.9924 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 34.992381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 29.59,
      "GSM8K": 4.1,
      "DC-Homograph": 50.93,
      "MC-Homograph": 52.53,
      "PiQA": 54.05,
      "Proverb-Quiz": 28.65,
      "VerbEval": 26.11,
      "Winogrande": 49.07,
      "Arc-Challenge": 37.5,
      "Arc-Easy": 47.38,
      "Feqh": 31.43,
      "Hallucination (Truthfulness)": 3.34,
      "P-Hellaswag": 55.4,
      "Law": 24,
      "AUT Multiple Choice": 29.9,
      "Parsi Literature": 27.03,
      "BoolQA": 64.1,
      "Reading Comprehension": 38,
      "PartExpert": 28.59,
      "MMLU Pro": 15.7,
      "Iranian Social Norms": 37.44
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "mimic-cdm-avg4",
    "score": 0.7048,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm' row 'google/gemma-3-27b-it' (16-bit weights, context 8192); published average 70.48 = mean of {'Appendicitis': 89.9, 'Cholecystitis': 65.7, 'Diverticulitis': 66.1, 'Pancreatitis': 60.2} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 70.48,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-12b",
    "bench": "mimic-cdm-avg4",
    "score": 0.662,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm' row 'google/gemma-3-12b-it' (16-bit weights, context 8192); published average 66.2 = mean of {'Appendicitis': 86.1, 'Cholecystitis': 59.4, 'Diverticulitis': 61.1, 'Pancreatitis': 58.2} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 66.2,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-4b",
    "bench": "mimic-cdm-avg4",
    "score": 0.508,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm' row 'google/gemma-3-4b-it' (16-bit weights, context 8192); published average 50.8 = mean of {'Appendicitis': 64.3, 'Cholecystitis': 55.7, 'Diverticulitis': 36.2, 'Pancreatitis': 47.0} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 50.8,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-1b",
    "bench": "mimic-cdm-avg4",
    "score": 0.2145,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm' row 'google/gemma-3-1b-it' (16-bit weights, context 8192); published average 21.45 = mean of {'Appendicitis': 45.2, 'Cholecystitis': 15.9, 'Diverticulitis': 12.8, 'Pancreatitis': 11.9} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 21.45,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-3-70b",
    "bench": "mimic-cdm-fi-avg4",
    "score": 0.8385,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm_fi' row 'meta-llama/Llama-3.3-70B-Instruct 16bit' (16-bit weights, context 8192); published average 83.85 = mean of {'Appendicitis': 97.9, 'Cholecystitis': 82.6, 'Diverticulitis': 72.0, 'Pancreatitis': 82.9} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 83.85,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-27b",
    "bench": "mimic-cdm-fi-avg4",
    "score": 0.7555,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm_fi' row 'google/gemma-3-27b-it' (16-bit weights, context 8192); published average 75.55 = mean of {'Appendicitis': 97.9, 'Cholecystitis': 73.5, 'Diverticulitis': 72.4, 'Pancreatitis': 58.4} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 75.55,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-12b",
    "bench": "mimic-cdm-fi-avg4",
    "score": 0.7438,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm_fi' row 'google/gemma-3-12b-it' (16-bit weights, context 8192); published average 74.38 = mean of {'Appendicitis': 96.5, 'Cholecystitis': 70.7, 'Diverticulitis': 59.1, 'Pancreatitis': 71.2} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 74.38,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-4b",
    "bench": "mimic-cdm-fi-avg4",
    "score": 0.694,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm_fi' row 'google/gemma-3-4b-it' (16-bit weights, context 8192); published average 69.4 = mean of {'Appendicitis': 92.7, 'Cholecystitis': 69.0, 'Diverticulitis': 53.3, 'Pancreatitis': 62.6} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 69.4,
    "scale_divisor": 100
  },
  {
    "model": "gemma-3-1b",
    "bench": "mimic-cdm-fi-avg4",
    "score": 0.2188,
    "source": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json",
    "source_id": "mimic-cdm-leaderboard",
    "by": "independent",
    "date": "2025-07-04",
    "note": "MIMIC-CDM leaderboard 'cdm_fi' row 'google/gemma-3-1b-it' (16-bit weights, context 8192); published average 21.88 = mean of {'Appendicitis': 37.8, 'Cholecystitis': 14.7, 'Diverticulitis': 9.7, 'Pancreatitis': 25.3} (divided by 100). Quantised rows of other models are not imported.",
    "raw_value": 21.88,
    "scale_divisor": 100
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "libra-mini-2605",
    "score": 0.355442,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-0.8B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-0.8B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.5605, ru_sci_passage_count 0.1779, ru_2wikimultihopqa 0.4600, long_context_multiq 0.3173, librusec_mhqa 0.3521, ru_babilong_qa3 0.2650) = 0.3554, as computed by the Space's Mini table. The file's own 'total_score' (0.27) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.560528,
      "ru_sci_passage_count": 0.177869,
      "ru_2wikimultihopqa": 0.459974,
      "long_context_multiq": 0.317264,
      "librusec_mhqa": 0.352055,
      "ru_babilong_qa3": 0.264962
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "libra-mini-2605",
    "score": 0.374036,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-2B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-2B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.5898, ru_sci_passage_count 0.1872, ru_2wikimultihopqa 0.4840, long_context_multiq 0.3339, librusec_mhqa 0.3705, ru_babilong_qa3 0.2788) = 0.3740, as computed by the Space's Mini table. The file's own 'total_score' (0.34) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.58985,
      "ru_sci_passage_count": 0.187173,
      "ru_2wikimultihopqa": 0.484035,
      "long_context_multiq": 0.333861,
      "librusec_mhqa": 0.370472,
      "ru_babilong_qa3": 0.278823
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "libra-mini-2605",
    "score": 0.390986,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-4B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-4B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.6166, ru_sci_passage_count 0.1957, ru_2wikimultihopqa 0.5060, long_context_multiq 0.3490, librusec_mhqa 0.3873, ru_babilong_qa3 0.2915) = 0.3910, as computed by the Space's Mini table. The file's own 'total_score' (0.43) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.616581,
      "ru_sci_passage_count": 0.195656,
      "ru_2wikimultihopqa": 0.505971,
      "long_context_multiq": 0.348991,
      "librusec_mhqa": 0.387261,
      "ru_babilong_qa3": 0.291459
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "libra-mini-2605",
    "score": 0.414993,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-9B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-9B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.6544, ru_sci_passage_count 0.2077, ru_2wikimultihopqa 0.5370, long_context_multiq 0.3704, librusec_mhqa 0.4110, ru_babilong_qa3 0.3094) = 0.4150, as computed by the Space's Mini table. The file's own 'total_score' (0.51) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.654439,
      "ru_sci_passage_count": 0.207669,
      "ru_2wikimultihopqa": 0.537038,
      "long_context_multiq": 0.370419,
      "librusec_mhqa": 0.411039,
      "ru_babilong_qa3": 0.309354
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "libra-mini-2605",
    "score": 0.437364,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-35B-A3B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-35B-A3B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.6897, ru_sci_passage_count 0.2189, ru_2wikimultihopqa 0.5660, long_context_multiq 0.3904, librusec_mhqa 0.4332, ru_babilong_qa3 0.3260) = 0.4374, as computed by the Space's Mini table. The file's own 'total_score' (0.61) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.689719,
      "ru_sci_passage_count": 0.218864,
      "ru_2wikimultihopqa": 0.565989,
      "long_context_multiq": 0.390387,
      "librusec_mhqa": 0.433197,
      "ru_babilong_qa3": 0.326031
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "libra-mini-2605",
    "score": 0.427307,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3.5-122B-A10B.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3.5-122B-A10B.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.6739, ru_sci_passage_count 0.2138, ru_2wikimultihopqa 0.5530, long_context_multiq 0.3814, librusec_mhqa 0.4232, ru_babilong_qa3 0.3185) = 0.4273, as computed by the Space's Mini table. The file's own 'total_score' (0.65) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.673858,
      "ru_sci_passage_count": 0.213831,
      "ru_2wikimultihopqa": 0.552973,
      "long_context_multiq": 0.38141,
      "librusec_mhqa": 0.423236,
      "ru_babilong_qa3": 0.318534
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "libra-mini-2605",
    "score": 0.514906,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/Qwen_Qwen3-30B-A3B-Instruct-2507.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file Qwen_Qwen3-30B-A3B-Instruct-2507.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.8120, ru_sci_passage_count 0.2577, ru_2wikimultihopqa 0.6663, long_context_multiq 0.4596, librusec_mhqa 0.5100, ru_babilong_qa3 0.3838) = 0.5149, as computed by the Space's Mini table. The file's own 'total_score' (0.7126490740740741) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.8119999999999999,
      "ru_sci_passage_count": 0.25766666666666665,
      "ru_2wikimultihopqa": 0.6663333333333333,
      "long_context_multiq": 0.4596,
      "librusec_mhqa": 0.51,
      "ru_babilong_qa3": 0.3838333333333334
    }
  },
  {
    "model": "llama-3-1-8b",
    "bench": "libra-mini-2605",
    "score": 0.438722,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/meta-llama_Llama-3.1-8B-Instruct.json",
    "source_id": "libra-leaderboard",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file meta-llama_Llama-3.1-8B-Instruct.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.6802, ru_sci_passage_count 0.1322, ru_2wikimultihopqa 0.6477, long_context_multiq 0.4068, librusec_mhqa 0.4870, ru_babilong_qa3 0.2785) = 0.4387, as computed by the Space's Mini table. The file's own 'total_score' (0.6474074074074074) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Thinking setting not stated. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.6801999999999999,
      "ru_sci_passage_count": 0.13216666666666668,
      "ru_2wikimultihopqa": 0.6476666666666667,
      "long_context_multiq": 0.40679999999999994,
      "librusec_mhqa": 0.487,
      "ru_babilong_qa3": 0.2785
    }
  },
  {
    "model": "gpt-5",
    "bench": "rpc-bench-text-overall",
    "score": 0.3746,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'GPT-5' (OpenAI, text input, status published); overall 37.46 (correctness 69.1, completeness 67.33, conciseness 54.93, f1_like 68.2); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value."
  },
  {
    "model": "gpt-5-2",
    "bench": "rpc-bench-text-overall",
    "score": 0.3519,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'GPT-5.2' (OpenAI, text input, status published); overall 35.19 (correctness 66.84, completeness 64.03, conciseness 53.81, f1_like 65.4); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value."
  },
  {
    "model": "deepseek-v3-2",
    "bench": "rpc-bench-text-overall",
    "score": 0.3204,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'DeepSeek-V3.2' (DeepSeek-AI, text input, status published); overall 32.04 (correctness 58.73, completeness 55.19, conciseness 56.31, f1_like 56.91); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value."
  },
  {
    "model": "deepseek-v3-1",
    "bench": "rpc-bench-text-overall",
    "score": 0.3084,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'DeepSeek-V3.1' (DeepSeek-AI, text input, status published); overall 30.84 (correctness 57.85, completeness 54.85, conciseness 54.76, f1_like 56.31); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value."
  },
  {
    "model": "glm-4-7",
    "bench": "rpc-bench-text-overall",
    "score": 0.2881,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "lab",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'GLM-4.7' (Z.ai, text input, status published); overall 28.81 (correctness 54.36, completeness 51.75, conciseness 54.34, f1_like 53.02); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value. Z.ai evaluating its own model."
  },
  {
    "model": "glm-4-5",
    "bench": "rpc-bench-text-overall",
    "score": 0.2572,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "lab",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'GLM-4.5' (Z.ai, text input, status published); overall 25.72 (correctness 58.95, completeness 59.54, conciseness 43.41, f1_like 59.24); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value. Z.ai evaluating its own model."
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "rpc-bench-text-overall",
    "score": 0.2005,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "source_id": "zai-rpc-bench-leaderboard",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'Claude-Sonnet-4.5' (Anthropic, text input, status published); overall 20.05 (correctness 64.31, completeness 64.97, conciseness 31.02, f1_like 64.64); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.025735,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "source_id": "multi-swe-bench-experiments",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_Qwen2.5-72B-Instruct' (metadata verified: true) in each of the seven language 'verified' splits; resolved 42/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_Qwen2.5-72B-Instruct/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated.",
    "components": {
      "java": {
        "resolved": 14,
        "total": 128
      },
      "typescript": {
        "resolved": 10,
        "total": 224
      },
      "javascript": {
        "resolved": 3,
        "total": 356
      },
      "go": {
        "resolved": 6,
        "total": 428
      },
      "rust": {
        "resolved": 6,
        "total": 239
      },
      "c": {
        "resolved": 2,
        "total": 128
      },
      "c++": {
        "resolved": 1,
        "total": 129
      }
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.058824,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "source_id": "multi-swe-bench-experiments",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_DeepSeek-R1' (metadata verified: true) in each of the seven language 'verified' splits; resolved 96/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_DeepSeek-R1/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated.",
    "components": {
      "java": {
        "resolved": 29,
        "total": 128
      },
      "typescript": {
        "resolved": 14,
        "total": 224
      },
      "javascript": {
        "resolved": 16,
        "total": 356
      },
      "go": {
        "resolved": 16,
        "total": 428
      },
      "rust": {
        "resolved": 16,
        "total": 239
      },
      "c": {
        "resolved": 1,
        "total": 128
      },
      "c++": {
        "resolved": 4,
        "total": 129
      }
    }
  },
  {
    "model": "llama-4-maverick",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.061275,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "source_id": "multi-swe-bench-experiments",
    "by": "independent",
    "date": "2025-04-26",
    "note": "Multi-SWE-bench experiments, submission folder '20250426_MagentLess_Llama-4-Maverick' (metadata verified: true) in each of the seven language 'verified' splits; resolved 100/1632 summed from evaluation/<lang>/verified/20250426_MagentLess_Llama-4-Maverick/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated.",
    "components": {
      "java": {
        "resolved": 19,
        "total": 128
      },
      "typescript": {
        "resolved": 21,
        "total": 224
      },
      "javascript": {
        "resolved": 18,
        "total": 356
      },
      "go": {
        "resolved": 15,
        "total": 428
      },
      "rust": {
        "resolved": 15,
        "total": 239
      },
      "c": {
        "resolved": 5,
        "total": 128
      },
      "c++": {
        "resolved": 7,
        "total": 129
      }
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "aime-2024",
    "score": 0.917,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'AIME24', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "AIME24",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "91.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "beyondaime",
    "score": 0.65,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'BeyondAIME', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "BeyondAIME",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "65",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "gpqa-diamond",
    "score": 0.714,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'GPQA-D', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Knowledge",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "GPQA-D",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "71.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "ifeval-unspecified",
    "score": 0.858,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Instruction Following",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "IFEval",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "85.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "korbench",
    "score": 0.706,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'KORBench', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "KORBench",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "70.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "mmlu",
    "score": 0.874,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'MMLU', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Knowledge",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "MMLU",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "mmmlu",
    "score": 0.784,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Multilingualism",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "MMMLU",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "multi-swe-bench",
    "score": 0.17,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'Multi-SWE-Bench', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agent",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "Multi-SWE-Bench",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "17",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "ruler-128k",
    "score": 0.946,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'RULER (128K)', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "RULER (128K)",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "94.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "simpleqa",
    "score": 0.097,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'SimpleQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Knowledge",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "SimpleQA",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "9.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "supergpqa",
    "score": 0.557,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Knowledge",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "SuperGPQA",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "55.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "tau-bench-airline",
    "score": 0.46,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agent",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "TAU1-Airline",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "46",
      "scale_divisor": 100
    }
  },
  {
    "model": "seed-oss-36b",
    "bench": "tau-bench-retail",
    "score": 0.704,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct",
    "by": "lab",
    "date": "2025-08-20",
    "mode": "thinking (unlimited budget default)",
    "note": "ByteDance Seed own evaluation in the pinned Seed-OSS-36B-Instruct model card, Seed-OSS-36B-Instruct table, own column. Generation configs: temperature=1.1, top_p=0.95; for Tau-bench temperature=1, top_p=0.7. ArcAGI-V2, AIR-Bench (safety) and the Agentless SWE row not used. Column 'Seed-OSS-36B-Instruct', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agent",
      "heading": "Seed-OSS-36B-Instruct",
      "row_label": "TAU1-Retail",
      "column_label": "Seed-OSS-36B-Instruct",
      "column_index": 6,
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "bfcl-v3-multiturn",
    "score": 0.37,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1 0528', row 'BFCL_v3_MultiTurn (Acc)', section 'Tools'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Tools",
      "heading": "DeepSeek-R1-0528",
      "row_label": "BFCL_v3_MultiTurn (Acc)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 3,
      "raw_value": "37.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "cnmo-2024",
    "score": 0.869,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1 0528', row 'CNMO 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math",
      "heading": "DeepSeek-R1-0528",
      "row_label": "CNMO 2024 (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 3,
      "raw_value": "86.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "frames",
    "score": 0.83,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1 0528', row 'FRAMES (Acc.)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "FRAMES (Acc.)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 3,
      "raw_value": "83.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "simpleqa",
    "score": 0.278,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1 0528', row 'SimpleQA (Correct)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "SimpleQA (Correct)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 3,
      "raw_value": "27.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "tau-bench-airline-gpt41-user",
    "score": 0.535,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Cell '53.5(Airline)/63.9(Retail)', Airline value; GPT-4.1 acts as the user.",
    "evidence": {
      "kind": "table-cell-manual",
      "quote": "53.5(Airline)/63.9(Retail)",
      "row_label": "Tau-Bench (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "raw_value": "53.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "tau-bench-retail-gpt41-user",
    "score": 0.639,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Cell '53.5(Airline)/63.9(Retail)', Retail value; GPT-4.1 acts as the user.",
    "evidence": {
      "kind": "table-cell-manual",
      "quote": "53.5(Airline)/63.9(Retail)",
      "row_label": "Tau-Bench (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "raw_value": "63.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "aider-polyglot",
    "score": 0.533,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'Aider-Polyglot (Acc.)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code",
      "heading": "DeepSeek-R1-0528",
      "row_label": "Aider-Polyglot (Acc.)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "53.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2024",
    "score": 0.798,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'AIME 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math",
      "heading": "DeepSeek-R1-0528",
      "row_label": "AIME 2024 (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "79.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2025",
    "score": 0.7,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'AIME 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math",
      "heading": "DeepSeek-R1-0528",
      "row_label": "AIME 2025 (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "cnmo-2024",
    "score": 0.788,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'CNMO 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math",
      "heading": "DeepSeek-R1-0528",
      "row_label": "CNMO 2024 (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "78.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "frames",
    "score": 0.825,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'FRAMES (Acc.)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "FRAMES (Acc.)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "82.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "hle",
    "score": 0.085,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'Humanity's Last Exam (Pass@1)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "Humanity's Last Exam (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "8.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.417,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'HMMT 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math",
      "heading": "DeepSeek-R1-0528",
      "row_label": "HMMT 2025 (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "41.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.635,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'LiveCodeBench (2408-2505) (Pass@1)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code",
      "heading": "DeepSeek-R1-0528",
      "row_label": "LiveCodeBench (2408-2505) (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "63.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "mmlu-redux",
    "score": 0.929,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'MMLU-Redux (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "MMLU-Redux (EM)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "92.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "simpleqa",
    "score": 0.301,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1-0528",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1-0528 model card (R1-0528 and same-lab DeepSeek R1 columns). Max generation length 64K tokens; temperature 0.6, top-p 0.95, 16 responses per query to estimate pass@1. SWE-Verified with the Agentless framework; HLE text-only prompts; GPT-4.1 as Tau-bench user. Column 'DeepSeek R1', row 'SimpleQA (Correct)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "DeepSeek-R1-0528",
      "row_label": "SimpleQA (Correct)",
      "column_label": "DeepSeek R1",
      "column_index": 2,
      "raw_value": "30.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "alpacaeval2-lc",
    "score": 0.876,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'AlpacaEval2.0 (LC-winrate)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "AlpacaEval2.0 (LC-winrate)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "87.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.923,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'ArenaHard (GPT-4-1106)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "ArenaHard (GPT-4-1106)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "92.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "c-eval",
    "score": 0.918,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'C-Eval (EM)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "C-Eval (EM)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "91.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "chinese-simpleqa",
    "score": 0.637,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'C-SimpleQA (Correct)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "C-SimpleQA (Correct)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "63.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "cluewsc",
    "score": 0.928,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'CLUEWSC (EM)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "CLUEWSC (EM)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "92.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "drop",
    "score": 0.922,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'DROP (3-shot F1)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "DROP (3-shot F1)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "92.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "ifeval-prompt-strict",
    "score": 0.833,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'IF-Eval (Prompt Strict)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "IF-Eval (Prompt Strict)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "83.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "livecodebench-unversioned",
    "score": 0.659,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'LiveCodeBench (Pass@1-COT)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Code",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "LiveCodeBench (Pass@1-COT)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "65.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "math-500",
    "score": 0.973,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'MATH-500 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MATH-500 (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "97.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1",
    "bench": "mmlu",
    "score": 0.908,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek R1', row 'MMLU (Pass@1)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MMLU (Pass@1)",
      "column_label": "DeepSeek R1",
      "column_index": 7,
      "raw_value": "90.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "aider-polyglot",
    "score": 0.496,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'Aider-Polyglot (Acc.)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Code",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "Aider-Polyglot (Acc.)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "49.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "aime-2024",
    "score": 0.392,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'AIME 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "AIME 2024 (Pass@1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "39.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "alpacaeval2-lc",
    "score": 0.7,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'AlpacaEval2.0 (LC-winrate)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "AlpacaEval2.0 (LC-winrate)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.855,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'ArenaHard (GPT-4-1106)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "ArenaHard (GPT-4-1106)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "85.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "c-eval",
    "score": 0.865,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'C-Eval (EM)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "C-Eval (EM)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "86.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "chinese-simpleqa",
    "score": 0.68,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'C-SimpleQA (Correct)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "C-SimpleQA (Correct)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "68.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "cluewsc",
    "score": 0.909,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'CLUEWSC (EM)', section 'Chinese'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chinese",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "CLUEWSC (EM)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "90.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "cnmo-2024",
    "score": 0.432,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'CNMO 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "CNMO 2024 (Pass@1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "43.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "drop",
    "score": 0.916,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'DROP (3-shot F1)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "DROP (3-shot F1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "91.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "frames",
    "score": 0.733,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'FRAMES (Acc.)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "FRAMES (Acc.)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "gpqa-diamond",
    "score": 0.591,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'GPQA-Diamond (Pass@1)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "GPQA-Diamond (Pass@1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "59.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "ifeval-prompt-strict",
    "score": 0.861,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'IF-Eval (Prompt Strict)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "IF-Eval (Prompt Strict)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "86.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "math-500",
    "score": 0.902,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'MATH-500 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Math",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MATH-500 (Pass@1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "90.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "mmlu",
    "score": 0.885,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'MMLU (Pass@1)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MMLU (Pass@1)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "88.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "mmlu-pro",
    "score": 0.759,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'MMLU-Pro (EM)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MMLU-Pro (EM)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "75.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "mmlu-redux",
    "score": 0.891,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'MMLU-Redux (EM)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "MMLU-Redux (EM)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "89.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "simpleqa",
    "score": 0.249,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'SimpleQA (Correct)', section 'English'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "English",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "SimpleQA (Correct)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "24.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "swe-bench-verified",
    "score": 0.42,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "source_id": "card-deepseek-ai--deepseek-r1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card, DeepSeek-R1-Evaluation table (DeepSeek R1 and same-lab DeepSeek V3 (Dec 2024) columns; DeepSeek V3 is a non-reasoning chat model). Max generation length 32,768 tokens; temperature 0.6, top-p 0.95, 64 responses per query to estimate pass@1. LiveCodeBench window not stated in the card. Column 'DeepSeek V3', row 'SWE Verified (Resolved)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Code",
      "heading": "DeepSeek-R1-Evaluation",
      "row_label": "SWE Verified (Resolved)",
      "column_label": "DeepSeek V3",
      "column_index": 4,
      "raw_value": "42.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "k2-horizon-7b",
    "bench": "browsecomp-context-managed",
    "score": 0.59,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high reasoning effort",
    "note": "IFM own evaluation in the pinned K2-Horizon-7B model card, Full Results table, own column (reference-model columns not used). All reported results use high reasoning effort. BrowseComp uses the Discard-all@95k context-length protocol from the DeepSeek-V3.2 report. LCR (long-context reasoning, AA-LCR style) not used. Column 'K2-Horizon-7B', row 'BrowseCompWeb browsing', section 'Agents'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agents",
      "heading": "Full Results",
      "row_label": "BrowseCompWeb browsing",
      "column_label": "K2-Horizon-7B",
      "column_index": 1,
      "raw_value": "59.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "k2-horizon-7b",
    "bench": "tau3-banking",
    "score": 0.258,
    "source": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md",
    "source_id": "card-ifm-k2-horizon-7b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high reasoning effort",
    "note": "IFM own evaluation in the pinned K2-Horizon-7B model card, Full Results table, own column (reference-model columns not used). All reported results use high reasoning effort. BrowseComp uses the Discard-all@95k context-length protocol from the DeepSeek-V3.2 report. LCR (long-context reasoning, AA-LCR style) not used. Column 'K2-Horizon-7B', row 'tau3-BankingAgentic tool use', section 'Agents'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agents",
      "heading": "Full Results",
      "row_label": "tau3-BankingAgentic tool use",
      "column_label": "K2-Horizon-7B",
      "column_index": 1,
      "raw_value": "25.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "browsecomp",
    "score": 0.799,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'BrowseComp', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Evaluation Results",
      "row_label": "BrowseComp",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "79.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "forte",
    "score": 0.732,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'FORTE ↗', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Evaluation Results",
      "row_label": "FORTE ↗",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "73.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "ifeval-unspecified",
    "score": 0.9,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'IFEval', section 'Foundational'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Foundational",
      "heading": "Evaluation Results",
      "row_label": "IFEval",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "imo-answerbench",
    "score": 0.818,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'IMO-AnswerBench', section 'Foundational'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Foundational",
      "heading": "Evaluation Results",
      "row_label": "IMO-AnswerBench",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "81.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "rwsearch",
    "score": 0.788,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'RWSearch ↗', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Evaluation Results",
      "row_label": "RWSearch ↗",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "78.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "swe-bench-multilingual",
    "score": 0.773,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'SWE-bench Multilingual', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Agent",
      "heading": "Evaluation Results",
      "row_label": "SWE-bench Multilingual",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "77.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "terminal-bench-2.1",
    "score": 0.708,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'Terminal-Bench 2.1', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Agent",
      "heading": "Evaluation Results",
      "row_label": "Terminal-Bench 2.1",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "70.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-2-0",
    "bench": "writingbench",
    "score": 0.838,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "source_id": "card-meituan-longcat--longcat-2.0",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Meituan LongCat own evaluation in the pinned LongCat-2.0 model card: all unstarred scores measured in-house under a unified harness (starred cells are other models' official reports and are not used). Column 'LongCat-2.0', row 'Writing Bench', section 'Foundational'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Foundational",
      "heading": "Evaluation Results",
      "row_label": "Writing Bench",
      "column_label": "LongCat-2.0",
      "column_index": 1,
      "raw_value": "83.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "aime-2024",
    "score": 0.753,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, Model Quality table (Eureka evaluation suite and internal benchmarks). Phi-4-reasoning-plus is not a catalogue checkpoint and is not used. Column 'Phi-4-reasoning', row 'AIME 24'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "AIME 24",
      "column_label": "Phi-4-reasoning",
      "column_index": 1,
      "raw_value": "75.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "aime-2025",
    "score": 0.629,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, Model Quality table (Eureka evaluation suite and internal benchmarks). Phi-4-reasoning-plus is not a catalogue checkpoint and is not used. Column 'Phi-4-reasoning', row 'AIME 25'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "AIME 25",
      "column_label": "Phi-4-reasoning",
      "column_index": 1,
      "raw_value": "62.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "arena-hard-unspecified",
    "score": 0.733,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4-reasoning', row 'ArenaHard'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "ArenaHard",
      "column_label": "Phi-4-reasoning",
      "column_index": 2,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "flenqa-3k",
    "score": 0.977,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4-reasoning', row 'FlenQA [3K-token subset]'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "FlenQA [3K-token subset]",
      "column_label": "Phi-4-reasoning",
      "column_index": 2,
      "raw_value": "97.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.658,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, Model Quality table (Eureka evaluation suite and internal benchmarks). Phi-4-reasoning-plus is not a catalogue checkpoint and is not used. Column 'Phi-4-reasoning', row 'GPQA-D'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "GPQA-D",
      "column_label": "Phi-4-reasoning",
      "column_index": 1,
      "raw_value": "65.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "humaneval-plus",
    "score": 0.929,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4-reasoning', row 'HumanEvalPlus'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "HumanEvalPlus",
      "column_label": "Phi-4-reasoning",
      "column_index": 2,
      "raw_value": "92.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "ifeval-strict-level-unspecified",
    "score": 0.834,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4-reasoning', row 'IFEval Strict'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "IFEval Strict",
      "column_label": "Phi-4-reasoning",
      "column_index": 2,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "livecodebench-20240801-20250201",
    "score": 0.538,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, Model Quality table (Eureka evaluation suite and internal benchmarks). Phi-4-reasoning-plus is not a catalogue checkpoint and is not used. Column 'Phi-4-reasoning', row 'LiveCodeBench (8/1/24–2/1/25)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "LiveCodeBench (8/1/24–2/1/25)",
      "column_label": "Phi-4-reasoning",
      "column_index": 1,
      "raw_value": "53.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "omni-math",
    "score": 0.766,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, Model Quality table (Eureka evaluation suite and internal benchmarks). Phi-4-reasoning-plus is not a catalogue checkpoint and is not used. Column 'Phi-4-reasoning', row 'OmniMath'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "OmniMath",
      "column_label": "Phi-4-reasoning",
      "column_index": 1,
      "raw_value": "76.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "arena-hard-unspecified",
    "score": 0.681,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "non-reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4', row 'ArenaHard'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "ArenaHard",
      "column_label": "Phi-4",
      "column_index": 1,
      "raw_value": "68.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "flenqa-3k",
    "score": 0.82,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "non-reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4', row 'FlenQA [3K-token subset]'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "FlenQA [3K-token subset]",
      "column_label": "Phi-4",
      "column_index": 1,
      "raw_value": "82.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "humaneval-plus",
    "score": 0.835,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "non-reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4', row 'HumanEvalPlus'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "HumanEvalPlus",
      "column_label": "Phi-4",
      "column_index": 1,
      "raw_value": "83.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "ifeval-strict-level-unspecified",
    "score": 0.623,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "non-reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4', row 'IFEval Strict'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "IFEval Strict",
      "column_label": "Phi-4",
      "column_index": 1,
      "raw_value": "62.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "mmlu-pro",
    "score": 0.715,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "source_id": "card-microsoft--phi-4-reasoning",
    "by": "lab",
    "date": "2025-04-09",
    "mode": "non-reasoning",
    "note": "Microsoft own evaluation in the pinned Phi-4-reasoning model card, second Model Quality table (Phi-4 column is a same-lab evaluation of the non-reasoning base chat model). Eureka evaluation suite. Column 'Phi-4', row 'MMLUPro'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Model Quality",
      "heading": "Model Quality",
      "row_label": "MMLUPro",
      "column_label": "Phi-4",
      "column_index": 1,
      "raw_value": "71.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "aime-2024",
    "score": 0.86,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'AIME 2024', section '*Mathematics*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Mathematics*",
      "heading": "2. Evaluation",
      "row_label": "AIME 2024",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "hle",
    "score": 0.084,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'HLE *(no tools)*', section '*Reasoning & Knowledge*'. Marker '\\*' in cell: footnote: conducted on the text-only HLE subset.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Reasoning & Knowledge*",
      "heading": "2. Evaluation",
      "row_label": "HLE *(no tools)*",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "8.4\\*",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.65,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'LiveCodeBench *(24/8~25/5)*', section '*General Coding*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*General Coding*",
      "heading": "2. Evaluation",
      "row_label": "LiveCodeBench *(24/8~25/5)*",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "65.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "longbench-v2",
    "score": 0.615,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'LongBench-v2', section '*Long Context*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Long Context*",
      "heading": "2. Evaluation",
      "row_label": "LongBench-v2",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "math-500",
    "score": 0.968,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'MATH-500', section '*Mathematics*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Mathematics*",
      "heading": "2. Evaluation",
      "row_label": "MATH-500",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "96.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "multichallenge",
    "score": 0.447,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'MultiChallenge', section '*General Assistant*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*General Assistant*",
      "heading": "2. Evaluation",
      "row_label": "MultiChallenge",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "44.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "simpleqa",
    "score": 0.185,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'SimpleQA', section '*Factuality*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Factuality*",
      "heading": "2. Evaluation",
      "row_label": "SimpleQA",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "18.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "tau-bench-airline-gpt41-user",
    "score": 0.62,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'TAU-bench *(airline)*', section '*Agentic Tool Use*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Agentic Tool Use*",
      "heading": "2. Evaluation",
      "row_label": "TAU-bench *(airline)*",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "tau-bench-retail-gpt41-user",
    "score": 0.635,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'TAU-bench *(retail)*', section '*Agentic Tool Use*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Agentic Tool Use*",
      "heading": "2. Evaluation",
      "row_label": "TAU-bench *(retail)*",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "63.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "minimax-m1",
    "bench": "zebralogic",
    "score": 0.868,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "source_id": "card-minimaxai--minimax-m1-80k",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking (80K extended thinking budget)",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card (MiniMax-M1-80K column; the 40K column is a different checkpoint). temperature=1.0, top_p=0.95. TAU-bench with GPT-4.1 as user model, no custom tools, max 40 interaction steps. Column 'MiniMax-M1-80K', row 'ZebraLogic', section '*Reasoning & Knowledge*'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "*Reasoning & Knowledge*",
      "heading": "2. Evaluation",
      "row_label": "ZebraLogic",
      "column_label": "MiniMax-M1-80K",
      "column_index": 2,
      "raw_value": "86.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "ministral-3-14b",
    "bench": "arena-hard-unspecified",
    "score": 0.551,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Ministral-3-14B-Instruct-2512 model card, Instruct table (rows for the 14B/8B/3B Instruct 2512 checkpoints; section is the Instruct evaluation, not Base or Reasoning). Arena Hard version/judge not stated; WildBench scale and MM MTBench (0-10) not used. Column 'Ministral 3 14B', row 'Arena Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Instruct",
      "heading": "Instruct",
      "row_label": "Arena Hard",
      "column_label": "Ministral 3 14B",
      "column_index": 1,
      "raw_value": "0.551",
      "scale_divisor": 1
    }
  },
  {
    "model": "ministral-3-3b",
    "bench": "arena-hard-unspecified",
    "score": 0.305,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Ministral-3-14B-Instruct-2512 model card, Instruct table (rows for the 14B/8B/3B Instruct 2512 checkpoints; section is the Instruct evaluation, not Base or Reasoning). Arena Hard version/judge not stated; WildBench scale and MM MTBench (0-10) not used. Column 'Ministral 3 3B', row 'Arena Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Instruct",
      "heading": "Instruct",
      "row_label": "Arena Hard",
      "column_label": "Ministral 3 3B",
      "column_index": 6,
      "raw_value": "0.305",
      "scale_divisor": 1
    }
  },
  {
    "model": "ministral-3-8b",
    "bench": "arena-hard-unspecified",
    "score": 0.509,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Ministral-3-14B-Instruct-2512 model card, Instruct table (rows for the 14B/8B/3B Instruct 2512 checkpoints; section is the Instruct evaluation, not Base or Reasoning). Arena Hard version/judge not stated; WildBench scale and MM MTBench (0-10) not used. Column 'Ministral 3 8B', row 'Arena Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Instruct",
      "heading": "Instruct",
      "row_label": "Arena Hard",
      "column_label": "Ministral 3 8B",
      "column_index": 4,
      "raw_value": "0.509",
      "scale_divisor": 1
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "chinese-simpleqa",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Instruct-2507', row 'CSimpleQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "CSimpleQA",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 6,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "simpleqa",
    "score": 0.543,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Instruct-2507', row 'SimpleQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SimpleQA",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 6,
      "raw_value": "54.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "zebralogic",
    "score": 0.95,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Instruct-2507', row 'ZebraLogic', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "ZebraLogic",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 6,
      "raw_value": "95.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "creative-writing-v3",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B-Thinking-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 7,
      "raw_value": "86.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hle",
    "score": 0.182,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B-Thinking-2507', row 'HLE', section 'Reasoning'. Marker '#' in cell: footnote: evaluated only on the HLE text-only subset (non-multimodal model).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HLE",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 7,
      "raw_value": "18.2#",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aime-2025",
    "score": 0.815,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'AIME25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "AIME25",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "81.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "arena-hard-v2",
    "score": 0.615,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'Arena-Hard v2$', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2$",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "bfcl-v3",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "70.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "creative-writing-v3",
    "score": 0.846,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "84.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "gpqa-unspecified",
    "score": 0.711,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "71.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "hle",
    "score": 0.118,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'HLE', section 'Reasoning'. Marker '#' in cell: footnote: evaluated only on the HLE text-only subset (non-multimodal model).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HLE",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "11.8#",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.625,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "62.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "ifeval-unspecified",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "include",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "78.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livebench-20241125",
    "score": 0.771,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "77.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livecodebench-v6",
    "score": 0.557,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'LiveCodeBench v6 (25.02-25.05)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "LiveCodeBench v6 (25.02-25.05)",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "55.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-pro",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'MMLU-Pro', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Pro",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-prox",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "80.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-redux",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "92.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "multi-if-unspecified",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "71.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "ojbench",
    "score": 0.256,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "25.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "polymath",
    "score": 0.547,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "54.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "supergpqa",
    "score": 0.607,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "60.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-airline",
    "score": 0.26,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "26.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-retail",
    "score": 0.548,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "54.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-airline",
    "score": 0.3,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-retail",
    "score": 0.404,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "40.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.219,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "21.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "writingbench",
    "score": 0.803,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-235B-A22B Thinking', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-235B-A22B Thinking",
      "column_index": 6,
      "raw_value": "80.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "zebralogic",
    "score": 0.9,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507",
    "by": "lab",
    "date": "2025-07-29",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'ZebraLogic', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "ZebraLogic",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 6,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "creative-writing-v3",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507",
    "by": "lab",
    "date": "2025-07-30",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab column 'Qwen3-235B-A22B Thinking' is the April 2025 hybrid model in thinking mode. Column 'Qwen3-30B-A3B-Thinking-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "84.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "aider-polyglot",
    "score": 0.129,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'Aider-Polyglot', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "Aider-Polyglot",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "12.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "arena-hard-v2",
    "score": 0.434,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "43.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.619,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "61.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "creative-writing-v3",
    "score": 0.835,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "83.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "gpqa-unspecified",
    "score": 0.62,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.31,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "31.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "include",
    "score": 0.601,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "60.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "livebench-20241125",
    "score": 0.63,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "63.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-prox",
    "score": 0.616,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "61.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.842,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "84.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "multi-if-unspecified",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "69.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "multipl-e",
    "score": 0.768,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'MultiPL-E', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "MultiPL-E",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "polymath",
    "score": 0.311,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "31.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.428,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "42.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "32.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau-bench-retail",
    "score": 0.487,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "48.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.24,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "24.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.404,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "40.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.132,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "13.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "writingbench",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "zebralogic",
    "score": 0.802,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "source_id": "card-qwen--qwen3-4b-instruct-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Instruct-2507', row 'ZebraLogic', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "ZebraLogic",
      "column_label": "Qwen3-4B-Instruct-2507",
      "column_index": 4,
      "raw_value": "80.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "arena-hard-v2",
    "score": 0.349,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'Arena-Hard v2$', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2$",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "34.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.712,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "71.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "creative-writing-v3",
    "score": 0.756,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "75.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "gpqa-unspecified",
    "score": 0.658,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "65.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.555,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "55.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.874,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "include",
    "score": 0.644,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "64.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.718,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "71.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.642,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "64.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "86.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.773,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "77.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "ojbench",
    "score": 0.179,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "17.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "polymath",
    "score": 0.462,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "46.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.478,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "47.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.48,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "48.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.661,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "66.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "58.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.535,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "53.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "writingbench",
    "score": 0.833,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "source_id": "card-qwen--qwen3-4b-thinking-2507",
    "by": "lab",
    "date": "2025-08-06",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-4B-Thinking-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-4B-Thinking-2507",
      "column_index": 3,
      "raw_value": "83.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aider-polyglot",
    "score": 0.573,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'Aider-Polyglot', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "Aider-Polyglot",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "57.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "arena-hard-v2",
    "score": 0.792,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "79.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.709,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "70.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "creative-writing-v3",
    "score": 0.875,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "87.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "gpqa-unspecified",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "77.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.554,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "55.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.887,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "88.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "include",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livebench-20241125",
    "score": 0.754,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-prox",
    "score": 0.794,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "79.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.931,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "93.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "multi-if-unspecified",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "77.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "multipl-e",
    "score": 0.879,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'MultiPL-E', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "MultiPL-E",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "87.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "polymath",
    "score": 0.502,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "50.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.626,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "62.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau-bench-airline",
    "score": 0.44,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "44.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau-bench-retail",
    "score": 0.713,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "71.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.5,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.325,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "32.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "writingbench",
    "score": 0.852,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-235B-A22B-Instruct-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-235B-A22B-Instruct-2507",
      "column_index": 3,
      "raw_value": "85.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "aider-polyglot",
    "score": 0.356,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'Aider-Polyglot', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "Aider-Polyglot",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "35.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "arena-hard-v2",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "69.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.651,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "65.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "creative-writing-v3",
    "score": 0.86,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "gpqa-unspecified",
    "score": 0.704,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.43,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "43.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.847,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "84.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "include",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "71.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "livebench-20241125",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "69.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-prox",
    "score": 0.72,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "72.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.893,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "89.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "multi-if-unspecified",
    "score": 0.679,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "67.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "multipl-e",
    "score": 0.838,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'MultiPL-E', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "MultiPL-E",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "83.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "polymath",
    "score": 0.431,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "43.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.534,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "53.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau-bench-airline",
    "score": 0.4,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "40.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau-bench-retail",
    "score": 0.591,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "59.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.38,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "38.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.57,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "57.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.123,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "12.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "writingbench",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-30B-A3B-Instruct-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-30B-A3B-Instruct-2507",
      "column_index": 1,
      "raw_value": "85.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "aider-polyglot",
    "score": 0.498,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'Aider-Polyglot', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "Aider-Polyglot",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "49.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "arena-hard-v2",
    "score": 0.827,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "82.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "bfcl-v3",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "creative-writing-v3",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'Creative Writing v3', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Creative Writing v3",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "85.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.729,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "72.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.541,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "54.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.876,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "87.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "include",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "livebench-20241125",
    "score": 0.758,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'LiveBench 20241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 20241125",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "mmlu-prox",
    "score": 0.767,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "76.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "mmlu-redux",
    "score": 0.909,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "90.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "multi-if-unspecified",
    "score": 0.758,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "multipl-e",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'MultiPL-E', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "MultiPL-E",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "87.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "polymath",
    "score": 0.459,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "45.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "supergpqa",
    "score": 0.588,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "58.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "tau-bench-airline",
    "score": 0.44,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "44.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "tau-bench-retail",
    "score": 0.609,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "60.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "tau2-airline",
    "score": 0.455,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "45.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "tau2-retail",
    "score": 0.573,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "57.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "tau2-telecom",
    "score": 0.132,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "13.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "writingbench",
    "score": 0.873,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "non-thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Non-thinking (instruct-only) checkpoints. Footnote: Arena-Hard v2 win rates judged by GPT-4.1. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Instruct-2507 and Qwen3-235B-A22B-Instruct-2507 (Qwen3-32B non-thinking column not used: hybrid model recorded in thinking mode only). Column 'Qwen3-Next-80B-A3B-Instruct', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-Next-80B-A3B-Instruct",
      "column_index": 4,
      "raw_value": "87.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "arena-hard-v2",
    "score": 0.797,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "79.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "71.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "gpqa-unspecified",
    "score": 0.811,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "81.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "87.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "include",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'LiveBench 241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 241125",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.938,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "93.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "80.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ojbench",
    "score": 0.325,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "32.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "polymath",
    "score": 0.601,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "60.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "64.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.46,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "46.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "58.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "71.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.456,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "45.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "writingbench",
    "score": 0.883,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-235B-A22B-Thinking-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-235B-A22B-Thinking-2507",
      "column_index": 3,
      "raw_value": "88.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "arena-hard-v2",
    "score": 0.56,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "56.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.724,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "72.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "gpqa-unspecified",
    "score": 0.734,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "73.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "71.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.889,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "88.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "include",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.768,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'LiveBench 241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 241125",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.764,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "76.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "91.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.764,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "76.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ojbench",
    "score": 0.251,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "25.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "polymath",
    "score": 0.526,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "52.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.568,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "56.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.48,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "48.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "58.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.588,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "58.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.263,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "26.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "writingbench",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-30B-A3B-Thinking-2507', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-30B-A3B-Thinking-2507",
      "column_index": 1,
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2025",
    "score": 0.729,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'AIME25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "AIME25",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "72.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "arena-hard-v2",
    "score": 0.484,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "48.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "bfcl-v3",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "gpqa-unspecified",
    "score": 0.684,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "68.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.515,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "51.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "ifeval-unspecified",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "include",
    "score": 0.737,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "73.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "livebench-20241125",
    "score": 0.749,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'LiveBench 241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 241125",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "74.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "livecodebench-v6",
    "score": 0.606,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'LiveCodeBench v6 (25.02-25.05)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "LiveCodeBench v6 (25.02-25.05)",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "60.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-pro",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'MMLU-Pro', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Pro",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "79.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-prox",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-redux",
    "score": 0.909,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "90.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "multi-if-unspecified",
    "score": 0.73,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "ojbench",
    "score": 0.241,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "24.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "polymath",
    "score": 0.474,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "47.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "supergpqa",
    "score": 0.541,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "54.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-airline",
    "score": 0.29,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "29.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-retail",
    "score": 0.528,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "52.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-airline",
    "score": 0.455,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "45.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-retail",
    "score": 0.497,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "49.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-telecom",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-32b",
    "bench": "writingbench",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-32B Thinking', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-32B Thinking",
      "column_index": 2,
      "raw_value": "79.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "arena-hard-v2",
    "score": 0.623,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'Arena-Hard v2*', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "Arena-Hard v2*",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "62.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "bfcl-v3",
    "score": 0.72,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'BFCL-v3', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "72.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.772,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'GPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "GPQA",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "77.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.739,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'HMMT25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "HMMT25",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "73.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.889,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'IFEval', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "IFEval",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "88.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "include",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "livebench-20241125",
    "score": 0.766,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'LiveBench 241125', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Performance",
      "row_label": "LiveBench 241125",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "76.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmlu-prox",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "78.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmlu-redux",
    "score": 0.925,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "92.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "multi-if-unspecified",
    "score": 0.778,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MultiIF', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "MultiIF",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "77.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "ojbench",
    "score": 0.297,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Performance",
      "row_label": "OJBench",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "29.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "polymath",
    "score": 0.563,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Performance",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "56.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "supergpqa",
    "score": 0.608,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Performance",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "60.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau-bench-airline",
    "score": 0.49,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU1-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Airline",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau-bench-retail",
    "score": 0.696,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU1-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU1-Retail",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "69.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau2-airline",
    "score": 0.605,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU2-Airline', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Airline",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "60.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau2-retail",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU2-Retail', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Retail",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau2-telecom",
    "score": 0.439,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU2-Telecom', section 'Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agent",
      "heading": "Performance",
      "row_label": "TAU2-Telecom",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "43.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "writingbench",
    "score": 0.846,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking/blob/e502dd4100cc68c0de57643fd4317ec93a128670/README.md",
    "source_id": "card-qwen--qwen3-next-80b-a3b-thinking",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned model card Performance table. Thinking-only checkpoints (Qwen3-32B / Qwen3-235B-A22B hybrid columns are their thinking mode). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1; for highly challenging tasks (PolyMATH and all reasoning and coding tasks) output length 81,920 tokens, otherwise 32,768; HLE text-only subset for non-multimodal models where marked. GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Same-lab columns for Qwen3-30B-A3B-Thinking-2507, Qwen3-32B (thinking) and Qwen3-235B-A22B-Thinking-2507. Column 'Qwen3-Next-80B-A3B-Thinking', row 'WritingBench', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment",
      "heading": "Performance",
      "row_label": "WritingBench",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 5,
      "raw_value": "84.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "ai2d-test",
    "score": 0.933,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "93.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "androidworld",
    "score": 0.664,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "66.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "bfcl-v4",
    "score": 0.722,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "72.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "browsecomp",
    "score": 0.638,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Browsecomp', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "63.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "browsecomp-zh",
    "score": 0.699,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Browsecomp-zh', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp-zh",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "69.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "c-eval",
    "score": 0.919,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "91.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "cc-ocr",
    "score": 0.818,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "81.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "charxiv-rq",
    "score": 0.772,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "77.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "countbench",
    "score": 0.97,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "97.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "deepplanning",
    "score": 0.241,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "24.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "dynamath",
    "score": 0.859,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "85.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "embspatialbench",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "erqa",
    "score": 0.62,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "fullstackbench-en",
    "score": 0.626,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'FullStackBench en', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench en",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "62.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "fullstackbench-zh",
    "score": 0.587,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'FullStackBench zh', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench zh",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "58.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "global-piqa",
    "score": 0.884,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "88.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hallusionbench",
    "score": 0.676,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "67.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hle-with-tools",
    "score": 0.475,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'HLE w/ tool', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "HLE w/ tool",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "47.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hmmt-feb-2025",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'HMMT Feb 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "91.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hmmt-nov-2025",
    "score": 0.903,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'HMMT Nov 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "90.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "ifbench-unspecified",
    "score": 0.761,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "76.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "ifeval-unspecified",
    "score": 0.934,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "93.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "include",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "lingoqa",
    "score": 0.808,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "80.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "longbench-v2",
    "score": 0.602,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "60.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "lvbench",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "math-vision",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "86.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mathvista-mini",
    "score": 0.874,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "maxife",
    "score": 0.879,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "87.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "medxpertqa-mm",
    "score": 0.673,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "67.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mlvu",
    "score": 0.873,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "87.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.928,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "92.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmlongbench-doc",
    "score": 0.59,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "59.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmlu-prox",
    "score": 0.822,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "82.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmlu-redux",
    "score": 0.94,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "94.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmmlu",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "86.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmmu",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmmu-pro",
    "score": 0.769,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "76.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmstar",
    "score": 0.829,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "82.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmvu",
    "score": 0.747,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "74.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "multichallenge",
    "score": 0.615,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mvbench",
    "score": 0.766,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "76.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "nova-63",
    "score": 0.586,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "58.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "ocrbench",
    "score": 0.921,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "92.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "ojbench",
    "score": 0.395,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "OJBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "39.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.898,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "89.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "osworld-verified",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "58.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "pmc-vqa",
    "score": 0.633,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "63.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "polymath",
    "score": 0.689,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "68.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "realworldqa",
    "score": 0.851,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "85.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "refspatialbench",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "screenspot-pro",
    "score": 0.704,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "seal-0",
    "score": 0.441,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Seal-0', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Seal-0",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "44.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "simplevqa",
    "score": 0.617,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "61.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "slake",
    "score": 0.816,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "81.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "supergpqa",
    "score": 0.671,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "67.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "tau2-bench",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "terminal-bench-2",
    "score": 0.494,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'Terminal Bench 2', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "Terminal Bench 2",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "49.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "videomme-with-sub",
    "score": 0.873,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "87.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "videomme-without-sub",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "videommmu",
    "score": 0.82,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "82.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "vita-bench",
    "score": 0.336,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "33.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "vlmsareblind",
    "score": 0.967,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'VlmsAreBlind', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "96.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "widesearch",
    "score": 0.605,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'WideSearch', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "60.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "wmt24pp",
    "score": 0.783,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "78.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "zerobench-sub",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-122B-A10B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-122B-A10B",
      "column_index": 4,
      "raw_value": "36.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ai2d-test",
    "score": 0.929,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "92.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "androidworld",
    "score": 0.642,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "64.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "bfcl-v4",
    "score": 0.685,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "68.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "browsecomp",
    "score": 0.61,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Browsecomp', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "61.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "browsecomp-zh",
    "score": 0.621,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Browsecomp-zh', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp-zh",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "62.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "c-eval",
    "score": 0.905,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "90.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "cc-ocr",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "charxiv-rq",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "countbench",
    "score": 0.978,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "97.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "deepplanning",
    "score": 0.226,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "22.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "dynamath",
    "score": 0.877,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "87.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "embspatialbench",
    "score": 0.845,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "84.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "erqa",
    "score": 0.605,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "60.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "fullstackbench-en",
    "score": 0.601,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'FullStackBench en', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench en",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "60.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "fullstackbench-zh",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'FullStackBench zh', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench zh",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "57.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "global-piqa",
    "score": 0.875,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "87.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hallusionbench",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hle-with-tools",
    "score": 0.485,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'HLE w/ tool', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "HLE w/ tool",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "48.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hmmt-feb-2025",
    "score": 0.92,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'HMMT Feb 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "92.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hmmt-nov-2025",
    "score": 0.898,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'HMMT Nov 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "89.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ifbench-unspecified",
    "score": 0.765,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "76.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ifeval-unspecified",
    "score": 0.95,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "95.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "include",
    "score": 0.816,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "81.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lingoqa",
    "score": 0.82,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "82.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "longbench-v2",
    "score": 0.606,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "60.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lvbench",
    "score": 0.736,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "73.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "math-vision",
    "score": 0.86,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mathvista-mini",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "87.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "maxife",
    "score": 0.88,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "88.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "medxpertqa-mm",
    "score": 0.624,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mlvu",
    "score": 0.859,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "85.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlongbench-doc",
    "score": 0.602,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "60.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-prox",
    "score": 0.822,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "82.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-redux",
    "score": 0.932,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "93.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmmlu",
    "score": 0.859,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "85.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmmu",
    "score": 0.823,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "82.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmmu-pro",
    "score": 0.75,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "75.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmstar",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmvu",
    "score": 0.733,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "multichallenge",
    "score": 0.608,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "60.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mvbench",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "nova-63",
    "score": 0.581,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "58.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ocrbench",
    "score": 0.894,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "89.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ojbench",
    "score": 0.401,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "OJBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "40.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.889,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "88.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "osworld-verified",
    "score": 0.562,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "56.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "pmc-vqa",
    "score": 0.624,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "polymath",
    "score": 0.712,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "71.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "realworldqa",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "83.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "refspatialbench",
    "score": 0.677,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "67.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "screenspot-pro",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "seal-0",
    "score": 0.472,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Seal-0', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Seal-0",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "47.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "simplevqa",
    "score": 0.56,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "56.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "slake",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "80.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "supergpqa",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "65.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "tau2-bench",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "79.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "terminal-bench-2",
    "score": 0.416,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'Terminal Bench 2', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "Terminal Bench 2",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "41.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "videomme-with-sub",
    "score": 0.87,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "87.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "videomme-without-sub",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "videommmu",
    "score": 0.823,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "82.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "vita-bench",
    "score": 0.419,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "41.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "vlmsareblind",
    "score": 0.969,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'VlmsAreBlind', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "96.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "widesearch",
    "score": 0.611,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'WideSearch', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "61.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "wmt24pp",
    "score": 0.776,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "77.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "zerobench-sub",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-27B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-27B",
      "column_index": 5,
      "raw_value": "36.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ai2d-test",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "androidworld",
    "score": 0.711,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "71.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "bfcl-v4",
    "score": 0.673,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "67.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "browsecomp",
    "score": 0.61,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Browsecomp', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "61.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "browsecomp-zh",
    "score": 0.695,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Browsecomp-zh', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Browsecomp-zh",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "69.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "c-eval",
    "score": 0.902,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "90.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "cc-ocr",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "80.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "charxiv-rq",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "77.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "countbench",
    "score": 0.978,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "97.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "deepplanning",
    "score": 0.228,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "22.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "dynamath",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "embspatialbench",
    "score": 0.831,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "83.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "erqa",
    "score": 0.648,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "64.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "fullstackbench-en",
    "score": 0.581,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'FullStackBench en', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench en",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "58.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "fullstackbench-zh",
    "score": 0.55,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'FullStackBench zh', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "FullStackBench zh",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "55.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "global-piqa",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hallusionbench",
    "score": 0.679,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "67.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hle-with-tools",
    "score": 0.474,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'HLE w/ tool', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "HLE w/ tool",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "47.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hmmt-feb-2025",
    "score": 0.89,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'HMMT Feb 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "89.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hmmt-nov-2025",
    "score": 0.892,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'HMMT Nov 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "89.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ifbench-unspecified",
    "score": 0.702,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "70.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.919,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "91.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "include",
    "score": 0.797,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "79.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "lingoqa",
    "score": 0.792,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "79.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "longbench-v2",
    "score": 0.59,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "59.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "lvbench",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "71.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "math-vision",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mathvista-mini",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "86.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "maxife",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "medxpertqa-mm",
    "score": 0.614,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "61.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mlvu",
    "score": 0.856,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "85.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.915,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "91.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlongbench-doc",
    "score": 0.595,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "59.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-prox",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-redux",
    "score": 0.933,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "93.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmmlu",
    "score": 0.852,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "85.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmmu",
    "score": 0.814,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "81.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmmu-pro",
    "score": 0.751,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "75.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmstar",
    "score": 0.819,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "81.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmvu",
    "score": 0.723,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "72.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "multichallenge",
    "score": 0.6,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "60.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mvbench",
    "score": 0.748,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "74.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "nova-63",
    "score": 0.571,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "57.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ocrbench",
    "score": 0.91,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "91.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ojbench",
    "score": 0.36,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'OJBench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "OJBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "36.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.893,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "89.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "osworld-verified",
    "score": 0.545,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "54.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "pmc-vqa",
    "score": 0.62,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "polymath",
    "score": 0.644,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "64.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "realworldqa",
    "score": 0.841,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "84.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "refspatialbench",
    "score": 0.635,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "63.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "screenspot-pro",
    "score": 0.686,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "68.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "seal-0",
    "score": 0.414,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Seal-0', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "Seal-0",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "41.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "simplevqa",
    "score": 0.583,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "58.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "slake",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "78.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "supergpqa",
    "score": 0.634,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "63.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "tau2-bench",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "81.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "terminal-bench-2",
    "score": 0.405,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'Terminal Bench 2', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Language",
      "row_label": "Terminal Bench 2",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "40.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "videomme-with-sub",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "videomme-without-sub",
    "score": 0.825,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "82.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "videommmu",
    "score": 0.804,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "80.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "vita-bench",
    "score": 0.319,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "31.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "vlmsareblind",
    "score": 0.97,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'VlmsAreBlind', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "97.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "widesearch",
    "score": 0.571,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'WideSearch', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "57.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "wmt24pp",
    "score": 0.763,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "76.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "zerobench-sub",
    "score": 0.341,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "source_id": "card-qwen--qwen3.5-122b-a10b",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-35B-A3B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-35B-A3B",
      "column_index": 6,
      "raw_value": "34.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "c-eval",
    "score": 0.681,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'C-Eval', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "68.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "global-piqa",
    "score": 0.631,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'Global PIQA', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "63.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-unspecified",
    "score": 0.401,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'GPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "GPQA",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "40.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "hmmt-feb-2025",
    "score": 0.102,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'HMMT Feb 25', section 'Reasoning (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (Thinking）",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "10.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "hmmt-nov-2025",
    "score": 0.089,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'HMMT Nov 25', section 'Reasoning (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (Thinking）",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "8.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifbench-unspecified",
    "score": 0.267,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'IFBench', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "26.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifeval-unspecified",
    "score": 0.725,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'IFEval', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "72.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "include",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'INCLUDE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "51.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "longbench-v2",
    "score": 0.265,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'LongBench v2', section 'Long Context (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context (Thinking）",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "26.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "maxife",
    "score": 0.507,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MAXIFE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "50.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-pro",
    "score": 0.565,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MMLU-Pro', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "MMLU-Pro",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "56.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-prox",
    "score": 0.494,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MMLU-ProX', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "49.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-redux",
    "score": 0.739,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MMLU-Redux', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "73.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmmlu",
    "score": 0.57,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MMMLU', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "57.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "multichallenge",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'MultiChallenge', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "nova-63",
    "score": 0.403,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'NOVA-63', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "40.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "polymath",
    "score": 0.252,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'PolyMATH', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "25.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "supergpqa",
    "score": 0.312,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'SuperGPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "31.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-1-7b",
    "bench": "wmt24pp",
    "score": 0.393,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3-1.7B', row 'WMT24++', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3-1.7B",
      "column_index": 2,
      "raw_value": "39.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "ai2d-test",
    "score": 0.699,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'. Cell '69.9/68.7': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "69.9/68.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "bfcl-v4",
    "score": 0.253,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'BFCL-V4', section 'General Agent (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent (Thinking）",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "25.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "c-eval",
    "score": 0.505,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'C-Eval', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "50.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "cc-ocr",
    "score": 0.632,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'CC-OCR', section 'Text Recognition and Document Understanding'. Cell '63.2/66.7': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "63.2/66.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "charxiv-rq",
    "score": 0.413,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'. Cell '41.3/38.2': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "41.3/38.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "countbench",
    "score": 0.77,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'CountBench', section 'Spatial Intelligence'. Cell '77.0/68.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "77.0/68.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "dynamath",
    "score": 0.499,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'DynaMath', section 'STEM and Puzzle'. Cell '49.9/46.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "49.9/46.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "embspatialbench",
    "score": 0.686,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'EmbSpatialBench', section 'Spatial Intelligence'. Cell '68.6/54.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "68.6/54.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "erqa",
    "score": 0.345,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'ERQA', section 'Spatial Intelligence'. Cell '34.5/23.8': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "34.5/23.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "global-piqa",
    "score": 0.594,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'Global PIQA', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "59.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "gpqa-unspecified",
    "score": 0.119,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'GPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "GPQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "11.9",
      "scale_divisor": 100
    },
    "invalid": "11.9% on GPQA (4 options, chance 25%) in thinking mode; probably unterminated thinking or unparsed answers rather than below-random knowledge. The next-lowest model on this benchmark scores 25.2%."
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "hallusionbench",
    "score": 0.531,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'HallusionBench', section 'General VQA'. Cell '53.1/46.7': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "53.1/46.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "ifbench-unspecified",
    "score": 0.21,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'IFBench', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "21.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "ifeval-unspecified",
    "score": 0.44,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'IFEval', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "44.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "include",
    "score": 0.406,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'INCLUDE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "40.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "longbench-v2",
    "score": 0.261,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'LongBench v2', section 'Long Context (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context (Thinking）",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "26.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "lvbench",
    "score": 0.451,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'LVBench', section 'Video Understanding'. Cell '45.1/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "45.1/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mathvista-mini",
    "score": 0.622,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'Mathvista(mini)', section 'STEM and Puzzle'. Cell '62.2/58.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "62.2/58.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "maxife",
    "score": 0.392,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MAXIFE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "39.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "medxpertqa-mm",
    "score": 0.171,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MedXpertQA-MM', section 'Medical VQA'. Cell '17.1/25.3': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "17.1/25.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mlvu",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MLVU', section 'Video Understanding'. Cell '65.6/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "65.6/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.699,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'. Cell '69.9/68.0': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "69.9/68.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmlongbench-doc",
    "score": 0.336,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'. Cell '33.6/28.1': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "33.6/28.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmlu-prox",
    "score": 0.346,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMLU-ProX', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "34.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmlu-redux",
    "score": 0.595,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMLU-Redux', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "59.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmmlu",
    "score": 0.443,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMMLU', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "44.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmmu",
    "score": 0.49,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMMU', section 'STEM and Puzzle'. Cell '49/47.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "49/47.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmmu-pro",
    "score": 0.312,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMMU-Pro', section 'STEM and Puzzle'. Cell '31.2/31.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "31.2/31.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmstar",
    "score": 0.583,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMStar', section 'General VQA'. Cell '58.3/55.9': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "58.3/55.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mmvu",
    "score": 0.343,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MMVU', section 'Video Understanding'. Cell '34.3/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "34.3/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "multichallenge",
    "score": 0.189,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MultiChallenge', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "18.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "mvbench",
    "score": 0.558,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'MVBench', section 'Video Understanding'. Cell '55.8/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "55.8/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "nova-63",
    "score": 0.424,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'NOVA-63', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "42.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "ocrbench",
    "score": 0.745,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'OCRBench', section 'Text Recognition and Document Understanding'. Cell '74.5/79.1': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "74.5/79.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.61,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'. Cell '61.0/70.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "61.0/70.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "pmc-vqa",
    "score": 0.404,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'PMC-VQA', section 'Medical VQA'. Cell '40.4/45.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "40.4/45.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "polymath",
    "score": 0.082,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'PolyMATH', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "8.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "realworldqa",
    "score": 0.634,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'RealWorldQA', section 'General VQA'. Cell '63.4/61.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "63.4/61.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "refspatialbench",
    "score": 0.235,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'RefSpatialBench', section 'Spatial Intelligence'. Cell '23.5/21.7': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "23.5/21.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "simplevqa",
    "score": 0.313,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'SimpleVQA', section 'General VQA'. Cell '31.3/30.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "31.3/30.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "slake",
    "score": 0.626,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'SLAKE', section 'Medical VQA'. Cell '62.6/59.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "62.6/59.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "supergpqa",
    "score": 0.213,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'SuperGPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "21.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "tau2-bench",
    "score": 0.116,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'TAU2-Bench', section 'General Agent (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent (Thinking）",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "11.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "videomme-with-sub",
    "score": 0.638,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'VideoMME(w sub.)', section 'Video Understanding'. Cell '63.8/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "63.8/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "videomme-without-sub",
    "score": 0.577,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'VideoMME(w/o sub.)', section 'Video Understanding'. Cell '57.7/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "57.7/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "videommmu",
    "score": 0.443,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'VideoMMMU', section 'Video Understanding'. Cell '44.3/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "44.3/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "vlmsareblind",
    "score": 0.594,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'VlmsAreBlind', section 'STEM and Puzzle'. Cell '59.4/57.3': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "59.4/57.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "wmt24pp",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'WMT24++', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "zerobench-sub",
    "score": 0.129,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-0.8B', row 'ZEROBench_sub', section 'STEM and Puzzle'. Cell '12.9/11.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-0.8B",
      "column_index": 4,
      "raw_value": "12.9/11.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ai2d-test",
    "score": 0.833,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'. Cell '83.3/81.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "83.3/81.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "bfcl-v4",
    "score": 0.436,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'BFCL-V4', section 'General Agent (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent (Thinking）",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "43.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "c-eval",
    "score": 0.732,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'C-Eval', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "73.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "cc-ocr",
    "score": 0.729,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'CC-OCR', section 'Text Recognition and Document Understanding'. Cell '72.9/75.8': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "72.9/75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "charxiv-rq",
    "score": 0.588,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'. Cell '58.8/52.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "58.8/52.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "countbench",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'CountBench', section 'Spatial Intelligence'. Cell '91.4/86.8': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "91.4/86.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "dynamath",
    "score": 0.736,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'DynaMath', section 'STEM and Puzzle'. Cell '73.6/69.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "73.6/69.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "embspatialbench",
    "score": 0.779,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'EmbSpatialBench', section 'Spatial Intelligence'. Cell '77.9/66.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "77.9/66.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "erqa",
    "score": 0.438,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'ERQA', section 'Spatial Intelligence'. Cell '43.8/33.0': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "43.8/33.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "global-piqa",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'Global PIQA', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "gpqa-unspecified",
    "score": 0.516,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'GPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "GPQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "51.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hallusionbench",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'HallusionBench', section 'General VQA'. Cell '58.0/51.3': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "58.0/51.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hmmt-feb-2025",
    "score": 0.229,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'HMMT Feb 25', section 'Reasoning (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (Thinking）",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "22.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hmmt-nov-2025",
    "score": 0.196,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'HMMT Nov 25', section 'Reasoning (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (Thinking）",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "19.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ifbench-unspecified",
    "score": 0.413,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'IFBench', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "41.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ifeval-unspecified",
    "score": 0.786,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'IFEval', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "78.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "include",
    "score": 0.554,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'INCLUDE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "55.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "longbench-v2",
    "score": 0.387,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'LongBench v2', section 'Long Context (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context (Thinking）",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "38.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "lvbench",
    "score": 0.571,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'LVBench', section 'Video Understanding'. Cell '57.1/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "57.1/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mathvista-mini",
    "score": 0.767,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'Mathvista(mini)', section 'STEM and Puzzle'. Cell '76.7/73.9': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "76.7/73.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "maxife",
    "score": 0.606,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MAXIFE', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "60.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "medxpertqa-mm",
    "score": 0.269,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MedXpertQA-MM', section 'Medical VQA'. Cell '26.9/19.1': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "26.9/19.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mlvu",
    "score": 0.762,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MLVU', section 'Video Understanding'. Cell '76.2/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "76.2/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.833,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'. Cell '83.3/81.3': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "83.3/81.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlongbench-doc",
    "score": 0.454,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'. Cell '45.4/38.8': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "45.4/38.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-prox",
    "score": 0.523,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMLU-ProX', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "52.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-redux",
    "score": 0.796,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMLU-Redux', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "79.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmmlu",
    "score": 0.631,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMMLU', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "63.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmmu",
    "score": 0.642,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMMU', section 'STEM and Puzzle'. Cell '64.2/64.2': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "64.2/64.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmmu-pro",
    "score": 0.503,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMMU-Pro', section 'STEM and Puzzle'. Cell '50.3/47.7': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "50.3/47.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmstar",
    "score": 0.717,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMStar', section 'General VQA'. Cell '71.7/68.0': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "71.7/68.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmvu",
    "score": 0.486,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MMVU', section 'Video Understanding'. Cell '48.6/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "48.6/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "multichallenge",
    "score": 0.337,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MultiChallenge', section 'Instruction Following (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following (Thinking）",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "33.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mvbench",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'MVBench', section 'Video Understanding'. Cell '64.9/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "64.9/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "nova-63",
    "score": 0.464,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'NOVA-63', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "46.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ocrbench",
    "score": 0.845,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'OCRBench', section 'Text Recognition and Document Understanding'. Cell '84.5/85.4': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "84.5/85.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.798,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'. Cell '79.8/80.9': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "79.8/80.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "pmc-vqa",
    "score": 0.488,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'PMC-VQA', section 'Medical VQA'. Cell '48.8/54.0': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "48.8/54.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "polymath",
    "score": 0.261,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'PolyMATH', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "26.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "realworldqa",
    "score": 0.745,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'RealWorldQA', section 'General VQA'. Cell '74.5/71.2': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "74.5/71.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "refspatialbench",
    "score": 0.329,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'RefSpatialBench', section 'Spatial Intelligence'. Cell '32.9/30.0': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "32.9/30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "simplevqa",
    "score": 0.385,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'SimpleVQA', section 'General VQA'. Cell '38.5/39.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "38.5/39.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "slake",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'SLAKE', section 'Medical VQA'. Cell '74.4/67.5': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "74.4/67.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "supergpqa",
    "score": 0.375,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'SuperGPQA', section 'Knowledge & STEM (Thinking)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM (Thinking)",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "37.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "tau2-bench",
    "score": 0.488,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'TAU2-Bench', section 'General Agent (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent (Thinking）",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "48.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "videomme-with-sub",
    "score": 0.756,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'VideoMME(w sub.)', section 'Video Understanding'. Cell '75.6/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "75.6/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "videomme-without-sub",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'VideoMME(w/o sub.)', section 'Video Understanding'. Cell '69.0/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "69.0/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "videommmu",
    "score": 0.621,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'VideoMMMU', section 'Video Understanding'. Cell '62.1/--': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "62.1/--",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "vlmsareblind",
    "score": 0.758,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'VlmsAreBlind', section 'STEM and Puzzle'. Cell '75.8/74.3': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "75.8/74.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "wmt24pp",
    "score": 0.458,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-2B/0.8B card, Thinking sections only (Instruct/Non-Thinking section not used). Qwen3-1.7B column is a same-lab evaluation of the hybrid model in thinking mode. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'WMT24++', section 'Multilingualism (Thinking）'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism (Thinking）",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "45.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-2b",
    "bench": "zerobench-sub",
    "score": 0.171,
    "source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
    "source_id": "card-qwen--qwen3.5-2b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-2B/0.8B card. Scores of Qwen3.5 small models: language table sections labelled by mode; only Thinking sections used. VL cells are 'Thinking / Non-thinking' per footnote, thinking (first) value used. Sampling: thinking temperature=0.6, top_p=0.95, top_k=20, presence_penalty=1.5 (video: temperature=1.0). TAU2 airline uses Claude Opus 4.5 system-card fixes. Column 'Qwen3.5-2B', row 'ZEROBench_sub', section 'STEM and Puzzle'. Cell '17.1/18.6': first value used (footnote: scores reported as Thinking / Non-thinking).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-2B",
      "column_index": 3,
      "raw_value": "17.1/18.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ai2d-test",
    "score": 0.939,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "93.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "androidworld",
    "score": 0.668,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "66.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "bfcl-v4",
    "score": 0.729,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "72.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "browsecomp",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'BrowseComp', section 'Search Agent³'. Cell '69.0/78.6': first value used (footnote: 69.0 with simple context-folding (256k), 78.6 with the discard-all strategy).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent³",
      "heading": "Language",
      "row_label": "BrowseComp",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "69.0/78.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "browsecomp-zh",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'BrowseComp-zh', section 'Search Agent³'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent³",
      "heading": "Language",
      "row_label": "BrowseComp-zh",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "c-eval",
    "score": 0.93,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "93.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "cc-ocr",
    "score": 0.82,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "82.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "charxiv-rq",
    "score": 0.808,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "80.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "countbench",
    "score": 0.972,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "97.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "deepplanning",
    "score": 0.343,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "34.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "dynamath",
    "score": 0.863,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "86.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "embspatialbench",
    "score": 0.845,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "84.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "erqa",
    "score": 0.675,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "67.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "global-piqa",
    "score": 0.898,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "89.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "gpqa-unspecified",
    "score": 0.884,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'GPQA', section 'STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM",
      "heading": "Language",
      "row_label": "GPQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "88.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hallusionbench",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "71.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle-with-tools",
    "score": 0.483,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'HLE w/ tool', section 'Search Agent³'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent³",
      "heading": "Language",
      "row_label": "HLE w/ tool",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "48.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hmmt-feb-2025",
    "score": 0.948,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'HMMT Feb 25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "94.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hmmt-nov-2025",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'HMMT Nov 25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "92.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ifbench-unspecified",
    "score": 0.765,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "76.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ifeval-unspecified",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "imo-answerbench",
    "score": 0.809,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'IMOAnswerBench', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Language",
      "row_label": "IMOAnswerBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "80.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "include",
    "score": 0.856,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "85.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "lingoqa",
    "score": 0.816,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "81.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "longbench-v2",
    "score": 0.632,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "63.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "lvbench",
    "score": 0.755,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "75.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "math-vision",
    "score": 0.886,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "88.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mathvista-mini",
    "score": 0.903,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "90.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "maxife",
    "score": 0.882,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "88.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mcpmark",
    "score": 0.461,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MCP-Mark', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCP-Mark",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "46.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "medxpertqa-mm",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mlvu",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MLVU (M-Avg)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU (M-Avg)",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "86.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.937,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "93.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlongbench-doc",
    "score": 0.615,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-prox",
    "score": 0.847,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "84.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-redux",
    "score": 0.949,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "94.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmmlu",
    "score": 0.885,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "88.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmmu",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmmu-pro",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "79.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmstar",
    "score": 0.838,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "83.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmvu",
    "score": 0.754,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "multichallenge",
    "score": 0.676,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "67.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mvbench",
    "score": 0.776,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "77.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "nova-63",
    "score": 0.591,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "59.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ocrbench",
    "score": 0.931,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "93.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.908,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "90.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "osworld-verified",
    "score": 0.622,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "62.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "pmc-vqa",
    "score": 0.642,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "64.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "polymath",
    "score": 0.733,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "realworldqa",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "83.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "refspatialbench",
    "score": 0.736,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "73.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "screenspot-pro",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "65.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "seal-0",
    "score": 0.469,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'Seal-0', section 'Search Agent³'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent³",
      "heading": "Language",
      "row_label": "Seal-0",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "46.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "seccodebench",
    "score": 0.683,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'SecCodeBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SecCodeBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "68.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "simplevqa",
    "score": 0.671,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "67.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "slake",
    "score": 0.799,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "79.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "supergpqa",
    "score": 0.704,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-multilingual",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'SWE-bench Multilingual', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau2-bench",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "86.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "terminal-bench-2",
    "score": 0.525,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'Terminal Bench 2', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Terminal Bench 2",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "52.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "toolathlon",
    "score": 0.383,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'Tool Decathlon', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "Tool Decathlon",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "38.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "videomme-with-sub",
    "score": 0.875,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "87.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "videomme-without-sub",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "83.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "videommmu",
    "score": 0.847,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "84.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "vita-bench",
    "score": 0.497,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "49.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "we-math",
    "score": 0.879,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'We-Math', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "We-Math",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "87.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "widesearch",
    "score": 0.74,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'WideSearch', section 'Search Agent³'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Search Agent³",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "74.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "wmt24pp",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table, own column. Thinking mode by default. Footnotes: TAU2 airline with Claude Opus 4.5 fixes; MCPMark GitHub MCP v0.30.3, Playwright responses truncated at 32k; search agents 256k context folding; WideSearch 256k no context management; MMLU-ProX 29 languages; WMT24++ 55 languages XCOMET-XXL; MAXIFE 23 settings. Column 'Qwen3.5-397B-A17B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "zerobench-sub",
    "score": 0.41,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "source_id": "card-qwen--qwen3.5-397b-a17b",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table, own column. MathVision fixed boxed prompt. Column 'Qwen3.5-397B-A17B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 6,
      "raw_value": "41.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "bfcl-v4",
    "score": 0.424,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "42.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "c-eval",
    "score": 0.874,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'C-Eval', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "deepplanning",
    "score": 0.049,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "4.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "global-piqa",
    "score": 0.802,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "80.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "hmmt-feb-2025",
    "score": 0.631,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'HMMT Feb 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "63.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "hmmt-nov-2025",
    "score": 0.738,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'HMMT Nov 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "73.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ifbench-unspecified",
    "score": 0.515,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "51.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "longbench-v2",
    "score": 0.448,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "44.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "maxife",
    "score": 0.774,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "77.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "69.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmmlu",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "multichallenge",
    "score": 0.465,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "46.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "nova-63",
    "score": 0.525,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "52.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-bench",
    "score": 0.419,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "41.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "vita-bench",
    "score": 0.141,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "14.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "wmt24pp",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-30BA3B-Thinking-2507', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3-30BA3B-Thinking-2507",
      "column_index": 4,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ai2d-test",
    "score": 0.896,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "89.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "androidworld",
    "score": 0.586,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "58.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v4",
    "score": 0.503,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "50.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "c-eval",
    "score": 0.851,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'C-Eval', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "85.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "cc-ocr",
    "score": 0.767,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "76.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "charxiv-rq",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "70.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "countbench",
    "score": 0.963,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "96.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "deepplanning",
    "score": 0.176,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "17.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "dynamath",
    "score": 0.833,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "83.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "embspatialbench",
    "score": 0.813,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "81.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "erqa",
    "score": 0.54,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "54.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "global-piqa",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hallusionbench",
    "score": 0.65,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "65.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hmmt-feb-2025",
    "score": 0.74,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'HMMT Feb 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "74.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hmmt-nov-2025",
    "score": 0.768,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'HMMT Nov 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifbench-unspecified",
    "score": 0.592,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "59.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifeval-unspecified",
    "score": 0.898,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "89.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "include",
    "score": 0.71,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "71.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "lingoqa",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "longbench-v2",
    "score": 0.5,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "lvbench",
    "score": 0.664,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "66.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "math-vision",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mathvista-mini",
    "score": 0.851,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "85.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "maxife",
    "score": 0.78,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "medxpertqa-mm",
    "score": 0.429,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "42.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mlvu",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.894,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "89.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlongbench-doc",
    "score": 0.542,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "54.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-prox",
    "score": 0.715,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "71.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-redux",
    "score": 0.888,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMLU-Redux', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "88.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmmlu",
    "score": 0.761,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "76.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmmu",
    "score": 0.776,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "77.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmmu-pro",
    "score": 0.663,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "66.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmstar",
    "score": 0.783,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "78.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmvu",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "64.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "multichallenge",
    "score": 0.49,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mvbench",
    "score": 0.712,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "71.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "nova-63",
    "score": 0.543,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "54.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ocrbench",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ojbench",
    "score": 0.241,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'OJBench', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "OJBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "24.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "86.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "osworld-verified",
    "score": 0.356,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "35.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "pmc-vqa",
    "score": 0.555,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "55.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "polymath",
    "score": 0.511,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "51.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "realworldqa",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "refspatialbench",
    "score": 0.546,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "54.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "screenspot-pro",
    "score": 0.603,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "60.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "simplevqa",
    "score": 0.434,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "43.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "slake",
    "score": 0.761,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "76.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "supergpqa",
    "score": 0.529,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'SuperGPQA', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "52.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "tau2-bench",
    "score": 0.799,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "79.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "videomme-with-sub",
    "score": 0.835,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "83.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "videomme-without-sub",
    "score": 0.769,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "76.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "videommmu",
    "score": 0.741,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "74.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "vita-bench",
    "score": 0.22,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "22.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "vlmsareblind",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'VlmsAreBlind', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "we-math",
    "score": 0.754,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'We-Math', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "We-Math",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "wmt24pp",
    "score": 0.666,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-4B",
      "column_index": 6,
      "raw_value": "66.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-4b",
    "bench": "zerobench-sub",
    "score": 0.263,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-4B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-4B",
      "column_index": 5,
      "raw_value": "26.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ai2d-test",
    "score": 0.902,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "90.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "androidworld",
    "score": 0.578,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "57.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v4",
    "score": 0.661,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "66.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "c-eval",
    "score": 0.882,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'C-Eval', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "88.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "cc-ocr",
    "score": 0.793,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'CC-OCR', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "79.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "charxiv-rq",
    "score": 0.73,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'CharXiv(RQ)', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv(RQ)",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "countbench",
    "score": 0.972,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "97.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "deepplanning",
    "score": 0.18,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "18.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "dynamath",
    "score": 0.836,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'DynaMath', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "83.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "embspatialbench",
    "score": 0.83,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "83.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "erqa",
    "score": 0.555,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "55.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "global-piqa",
    "score": 0.832,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "83.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "hallusionbench",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "hmmt-feb-2025",
    "score": 0.832,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'HMMT Feb 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "83.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "hmmt-nov-2025",
    "score": 0.829,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'HMMT Nov 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "82.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifbench-unspecified",
    "score": 0.645,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "64.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifeval-unspecified",
    "score": 0.915,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'IFEval', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFEval",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "91.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "include",
    "score": 0.756,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "75.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "lingoqa",
    "score": 0.804,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'LingoQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "LingoQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "80.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "longbench-v2",
    "score": 0.552,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "55.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "lvbench",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "math-vision",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MathVision', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVision",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mathvista-mini",
    "score": 0.857,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'Mathvista(mini)', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "Mathvista(mini)",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "85.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "maxife",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "medxpertqa-mm",
    "score": 0.499,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MedXpertQA-MM', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "MedXpertQA-MM",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "49.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mlvu",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "84.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.901,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "90.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlongbench-doc",
    "score": 0.577,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMLongBench-Doc', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "MMLongBench-Doc",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "57.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-prox",
    "score": 0.763,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "76.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-redux",
    "score": 0.911,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMLU-Redux', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "91.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmmlu",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "81.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmmu",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMMU', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmmu-pro",
    "score": 0.701,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMMU-Pro', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "70.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmstar",
    "score": 0.797,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "79.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmvu",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MMVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MMVU",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "multichallenge",
    "score": 0.545,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "54.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mvbench",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "nova-63",
    "score": 0.559,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "55.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ocrbench",
    "score": 0.892,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'OCRBench', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "89.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ojbench",
    "score": 0.292,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'OJBench', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "OJBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "29.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.877,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "87.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "osworld-verified",
    "score": 0.418,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'OSWorld-Verified', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "OSWorld-Verified",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "41.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "pmc-vqa",
    "score": 0.579,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'PMC-VQA', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "PMC-VQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "57.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "polymath",
    "score": 0.573,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "57.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "realworldqa",
    "score": 0.803,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "80.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "refspatialbench",
    "score": 0.585,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "58.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "screenspot-pro",
    "score": 0.652,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'ScreenSpot Pro', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "ScreenSpot Pro",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "65.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "simplevqa",
    "score": 0.512,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "51.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "slake",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'SLAKE', section 'Medical VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Medical VQA",
      "heading": "Vision Language",
      "row_label": "SLAKE",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "79.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "supergpqa",
    "score": 0.582,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'SuperGPQA', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "58.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "tau2-bench",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "79.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "videomme-with-sub",
    "score": 0.845,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "84.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "videomme-without-sub",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "videommmu",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "vita-bench",
    "score": 0.298,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "29.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "vlmsareblind",
    "score": 0.937,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'VlmsAreBlind', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "93.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "we-math",
    "score": 0.752,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'We-Math', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "We-Math",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "75.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "wmt24pp",
    "score": 0.726,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3.5-9B",
      "column_index": 5,
      "raw_value": "72.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-9b",
    "bench": "zerobench-sub",
    "score": 0.311,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Vision-language table of the Qwen3.5-9B/4B card. Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3.5-9B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.5-9B",
      "column_index": 4,
      "raw_value": "31.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "bfcl-v4",
    "score": 0.497,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'BFCL-V4', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "BFCL-V4",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "49.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "c-eval",
    "score": 0.897,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'C-Eval', section 'Knowledge & STEM'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge & STEM",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "89.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "deepplanning",
    "score": 0.004,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "0.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "global-piqa",
    "score": 0.835,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'Global PIQA', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "Global PIQA",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "83.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "hmmt-feb-2025",
    "score": 0.737,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'HMMT Feb 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "73.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "hmmt-nov-2025",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'HMMT Nov 25', section 'Reasoning & Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning & Coding",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "81.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "ifbench-unspecified",
    "score": 0.615,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'IFBench', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "IFBench",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "include",
    "score": 0.783,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'INCLUDE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "INCLUDE",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "78.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "longbench-v2",
    "score": 0.48,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'LongBench v2', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Long Context",
      "heading": "Language",
      "row_label": "LongBench v2",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "48.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "maxife",
    "score": 0.799,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MAXIFE', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MAXIFE",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "79.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmlu-prox",
    "score": 0.736,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MMLU-ProX', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMLU-ProX",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "73.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmmlu",
    "score": 0.813,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MMMLU', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "MMMLU",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "81.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "multichallenge",
    "score": 0.513,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'MultiChallenge', section 'Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following",
      "heading": "Language",
      "row_label": "MultiChallenge",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "51.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "nova-63",
    "score": 0.533,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'NOVA-63', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "NOVA-63",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "53.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "polymath",
    "score": 0.624,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'PolyMATH', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "PolyMATH",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "tau2-bench",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'TAU2-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU2-Bench",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "57.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "vita-bench",
    "score": 0.295,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "29.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "wmt24pp",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "source_id": "card-qwen--qwen3.5-9b",
    "by": "lab",
    "date": "2026-03-02",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.5 model card. Language table of the Qwen3.5-9B/4B card (Qwen-run columns for Qwen3-Next-80B-A3B-Thinking and Qwen3-30B-A3B-Thinking-2507 are same-lab evaluations). Card: Qwen3.5 models operate in thinking mode by default (table does not label mode). Footnotes: TAU2 airline uses the Claude Opus 4.5 system-card fixes; search agents use 256k context folding; WideSearch 256k without context management; MMLU-ProX averaged over 29 languages; WMT24++ harder subset, 55 languages, XCOMET-XXL; MAXIFE English + multilingual original prompts (23 settings); MathVision fixed boxed prompt for Qwen models. Column 'Qwen3-Next-80B-A3B-Thinking', row 'WMT24++', section 'Multilingualism'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingualism",
      "heading": "Language",
      "row_label": "WMT24++",
      "column_label": "Qwen3-Next-80B-A3B-Thinking",
      "column_index": 3,
      "raw_value": "57.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "aime-2026",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'AIME26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "AIME26",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "claw-eval-avg",
    "score": 0.643,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'Claw-Eval Avg', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Avg",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "64.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "claw-eval-pass3",
    "score": 0.462,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'Claw-Eval Pass^3', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Pass^3",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "46.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hmmt-feb-2026",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'HMMT Feb 26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 26",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "imo-answerbench",
    "score": 0.799,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'IMOAnswerBench', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "IMOAnswerBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "79.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "nl2repo",
    "score": 0.273,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'NL2Repo', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "NL2Repo",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "27.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "qwenclawbench",
    "score": 0.522,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'QwenClawBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "QwenClawBench",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "52.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "skillsbench-qwen-78",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'SkillsBench Avg5', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SkillsBench Avg5",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-multilingual",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'SWE-bench Multilingual', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.512,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-27B', row 'SWE-bench Pro', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Pro",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "51.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "claw-eval-avg",
    "score": 0.707,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'Claw-Eval Avg', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Avg",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "70.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "claw-eval-pass3",
    "score": 0.481,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'Claw-Eval Pass^3', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Pass^3",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "48.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hmmt-feb-2026",
    "score": 0.879,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'HMMT Feb 26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 26",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "87.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "nl2repo",
    "score": 0.322,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'NL2Repo', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "NL2Repo",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "32.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "qwenclawbench",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'QwenClawBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "QwenClawBench",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "51.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "skillsbench-qwen-78",
    "score": 0.3,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'SkillsBench Avg5', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SkillsBench Avg5",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.509,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.5-397B-A17B', row 'SWE-bench Pro', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Pro",
      "column_label": "Qwen3.5-397B-A17B",
      "column_index": 2,
      "raw_value": "50.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "androidworld",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'AndroidWorld', section 'Visual Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Visual Agent",
      "heading": "Vision Language",
      "row_label": "AndroidWorld",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "c-eval",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "91.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "cc-ocr",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'CC-OCR', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "81.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "charxiv-rq",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'CharXiv RQ', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv RQ",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "claw-eval-avg",
    "score": 0.724,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'Claw-Eval Avg', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Avg",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "72.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "claw-eval-pass3",
    "score": 0.606,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'Claw-Eval Pass^3', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Pass^3",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "60.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "countbench",
    "score": 0.978,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "97.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "dynamath",
    "score": 0.856,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'DynaMath', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "85.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "embspatialbench",
    "score": 0.846,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "84.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "erqa",
    "score": 0.625,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "62.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "hmmt-feb-2025",
    "score": 0.938,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'HMMT Feb 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "93.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "hmmt-feb-2026",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'HMMT Feb 26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 26",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "hmmt-nov-2025",
    "score": 0.907,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'HMMT Nov 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "90.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "imo-answerbench",
    "score": 0.808,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'IMOAnswerBench', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "IMOAnswerBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "80.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mathvista-mini",
    "score": 0.874,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MathVista mini', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVista mini",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mlvu",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.923,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "92.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmlu-redux",
    "score": 0.935,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "93.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmmu",
    "score": 0.829,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MMMU', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "82.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmmu-pro",
    "score": 0.758,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MMMU-Pro', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mmstar",
    "score": 0.814,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "81.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mvbench",
    "score": 0.755,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "75.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "nl2repo",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'NL2Repo', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "NL2Repo",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "36.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "ocrbench",
    "score": 0.894,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'OCRBench', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "89.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "qwenclawbench",
    "score": 0.534,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'QwenClawBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "QwenClawBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "53.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "realworldqa",
    "score": 0.841,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "84.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "refspatialbench",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "70.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "simplevqa",
    "score": 0.561,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "56.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "skillsbench-qwen-78",
    "score": 0.482,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'SkillsBench Avg5', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SkillsBench Avg5",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "48.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "supergpqa",
    "score": 0.66,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "66.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "swe-bench-multilingual",
    "score": 0.713,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'SWE-bench Multilingual', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "71.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.535,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'SWE-bench Pro', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Pro",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "53.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "terminal-bench-2",
    "score": 0.593,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-27B', row 'Terminal-Bench 2.0', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "59.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "videomme-with-sub",
    "score": 0.877,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "87.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "videommmu",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "84.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "vlmsareblind",
    "score": 0.97,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-27B', row 'VlmsAreBlind', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.6-27B",
      "column_index": 6,
      "raw_value": "97.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "aime-2026",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'AIME26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "AIME26",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "92.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "c-eval",
    "score": 0.9,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'C-Eval', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "C-Eval",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "cc-ocr",
    "score": 0.819,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'CC-OCR', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "CC-OCR",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "81.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "charxiv-rq",
    "score": 0.78,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'CharXiv RQ', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "CharXiv RQ",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "claw-eval-avg",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'Claw-Eval Avg', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Avg",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "68.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "claw-eval-pass3",
    "score": 0.5,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'Claw-Eval Pass^3', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Pass^3",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "countbench",
    "score": 0.961,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'CountBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "CountBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "96.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "dynamath",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'DynaMath', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "DynaMath",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "embspatialbench",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'EmbSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "EmbSpatialBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "erqa",
    "score": 0.618,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'ERQA', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "ERQA",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "61.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.86,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'GPQA Diamond', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "GPQA Diamond",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hmmt-feb-2025",
    "score": 0.907,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'HMMT Feb 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 25",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "90.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hmmt-feb-2026",
    "score": 0.836,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'HMMT Feb 26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 26",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "83.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hmmt-nov-2025",
    "score": 0.891,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'HMMT Nov 25', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Nov 25",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "89.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "imo-answerbench",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'IMOAnswerBench', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "IMOAnswerBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mathvista-mini",
    "score": 0.864,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MathVista mini', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MathVista mini",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "86.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mlvu",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MLVU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MLVU",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "86.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.928,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MMBenchEN-DEV-v1.1', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMBenchEN-DEV-v1.1",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "92.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmlu-redux",
    "score": 0.933,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'MMLU-Redux', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "MMLU-Redux",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "93.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmmu",
    "score": 0.817,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MMMU', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "81.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmmu-pro",
    "score": 0.753,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MMMU-Pro', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "MMMU-Pro",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "75.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmstar",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MMStar', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "MMStar",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "80.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mvbench",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'MVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "MVBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "nl2repo",
    "score": 0.294,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'NL2Repo', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "NL2Repo",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "29.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "ocrbench",
    "score": 0.9,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'OCRBench', section 'Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Document Understanding",
      "heading": "Vision Language",
      "row_label": "OCRBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "qwenclawbench",
    "score": 0.526,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'QwenClawBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "QwenClawBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "52.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "realworldqa",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'RealWorldQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "RealWorldQA",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "85.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "refspatialbench",
    "score": 0.643,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'RefSpatialBench', section 'Spatial Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Spatial Intelligence",
      "heading": "Vision Language",
      "row_label": "RefSpatialBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "64.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "simplevqa",
    "score": 0.589,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'SimpleVQA', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "SimpleVQA",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "58.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "skillsbench-qwen-78",
    "score": 0.287,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'SkillsBench Avg5', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SkillsBench Avg5",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "supergpqa",
    "score": 0.647,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'SuperGPQA', section 'Knowledge'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Knowledge",
      "heading": "Language",
      "row_label": "SuperGPQA",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "64.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.672,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'SWE-bench Multilingual', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "67.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.495,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'SWE-bench Pro', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Pro",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "49.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "terminal-bench-2",
    "score": 0.515,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. Column 'Qwen3.6-35B-A3B', row 'Terminal-Bench 2.0', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "51.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "videomme-with-sub",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'VideoMME(w sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w sub.)",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "videommmu",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'VideoMMMU', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMMMU",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "83.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "vlmsareblind",
    "score": 0.966,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "source_id": "card-qwen--qwen3.6-27b",
    "by": "lab",
    "date": "2026-04-22",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'VlmsAreBlind', section 'STEM & Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM & Puzzle",
      "heading": "Vision Language",
      "row_label": "VlmsAreBlind",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 5,
      "raw_value": "96.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mcp-atlas",
    "score": 0.684,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'MCP-Atlas', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCP-Atlas",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "68.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mcpmark",
    "score": 0.363,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'MCPMark', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCPMark",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "36.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "tau3-bench",
    "score": 0.684,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'TAU3-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU3-Bench",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "68.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "toolathlon",
    "score": 0.315,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'Tool Decathlon', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "Tool Decathlon",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "31.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "vita-bench",
    "score": 0.418,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "41.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-27b",
    "bench": "widesearch",
    "score": 0.664,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-27B', row 'WideSearch', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-27B",
      "column_index": 1,
      "raw_value": "66.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "aime-2026",
    "score": 0.91,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'AIME26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "AIME26",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "91.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "claw-eval-avg",
    "score": 0.654,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'Claw-Eval Avg', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Avg",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "65.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "claw-eval-pass3",
    "score": 0.51,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'Claw-Eval Pass^3', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "Claw-Eval Pass^3",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "51.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hmmt-feb-2026",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'HMMT Feb 26', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "HMMT Feb 26",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "78.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "imo-answerbench",
    "score": 0.768,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'IMOAnswerBench', section 'STEM & Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "STEM & Reasoning",
      "heading": "Language",
      "row_label": "IMOAnswerBench",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mcp-atlas",
    "score": 0.624,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'MCP-Atlas', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCP-Atlas",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mcpmark",
    "score": 0.27,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'MCPMark', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCPMark",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "27.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "nl2repo",
    "score": 0.205,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'NL2Repo', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "NL2Repo",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "20.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "qwenclawbench",
    "score": 0.477,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'QwenClawBench', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "QwenClawBench",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "47.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "skillsbench-qwen-78",
    "score": 0.044,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'SkillsBench Avg5', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SkillsBench Avg5",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "4.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.603,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'SWE-bench Multilingual', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "60.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.446,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'SWE-bench Pro', section 'Coding Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding Agent",
      "heading": "Language",
      "row_label": "SWE-bench Pro",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "44.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "tau3-bench",
    "score": 0.689,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'TAU3-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU3-Bench",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "68.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "toolathlon",
    "score": 0.287,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'Tool Decathlon', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "Tool Decathlon",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "vita-bench",
    "score": 0.291,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "29.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "widesearch",
    "score": 0.591,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.5-35BA3B', row 'WideSearch', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.5-35BA3B",
      "column_index": 3,
      "raw_value": "59.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "ai2d-test",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'AI2D_TEST', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "AI2D_TEST",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "92.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "deepplanning",
    "score": 0.259,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'DeepPlanning', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "DeepPlanning",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "25.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hallusionbench",
    "score": 0.698,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'HallusionBench', section 'General VQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General VQA",
      "heading": "Vision Language",
      "row_label": "HallusionBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "69.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "lvbench",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'LVBench', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "LVBench",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "71.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mcp-atlas",
    "score": 0.628,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'MCP-Atlas', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCP-Atlas",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "62.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mcpmark",
    "score": 0.37,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'MCPMark', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "MCPMark",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "37.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.899,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'OmniDocBench1.5', section 'Text Recognition and Document Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Text Recognition and Document Understanding",
      "heading": "Vision Language",
      "row_label": "OmniDocBench1.5",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "89.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "tau3-bench",
    "score": 0.672,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'TAU3-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "TAU3-Bench",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "67.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "toolathlon",
    "score": 0.269,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'Tool Decathlon', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "Tool Decathlon",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "26.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "videomme-without-sub",
    "score": 0.825,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'VideoMME(w/o sub.)', section 'Video Understanding'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Video Understanding",
      "heading": "Vision Language",
      "row_label": "VideoMME(w/o sub.)",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "82.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "vita-bench",
    "score": 0.356,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'VITA-Bench', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "VITA-Bench",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "35.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "widesearch",
    "score": 0.601,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Qwen3.6 card; thinking mode default. Footnotes: SWE-bench series internal agent scaffold (bash + file-edit), temp=1.0, top_p=0.95, 200K context; SWE-bench Pro public set with problematic tasks corrected (refined benchmark); Terminal-Bench 2.0 Harbor/Terminus-2, 3h timeout, temp=1.0, top_p=0.95, top_k=20, max_tokens=80K, avg of 5 runs; SkillsBench via OpenCode on 78 self-contained tasks, avg of 5; QwenClawBench internal, temp=0.6, 256K ctx; AIME 26 = full AIME 2026 I & II. TAU3-Bench: official user model gpt-5.2 (low effort) + BM25 retrieval; VITA-Bench avg subdomain scores, claude-4-sonnet judge; MCPMark GitHub MCP v0.30.3; MCP-Atlas public set, gemini-2.5-pro judge. Column 'Qwen3.6-35BA3B', row 'WideSearch', section 'General Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Agent",
      "heading": "Language",
      "row_label": "WideSearch",
      "column_label": "Qwen3.6-35BA3B",
      "column_index": 5,
      "raw_value": "60.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "zerobench-sub",
    "score": 0.344,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "source_id": "card-qwen--qwen3.6-35b-a3b",
    "by": "lab",
    "date": "2026-04-16",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.6 model card. Vision-language table; Qwen3.5 columns are Qwen's re-evaluations reported in the Qwen3.6 card. Column 'Qwen3.6-35B-A3B', row 'ZEROBench_sub', section 'STEM and Puzzle'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "STEM and Puzzle",
      "heading": "Vision Language",
      "row_label": "ZEROBench_sub",
      "column_label": "Qwen3.6-35B-A3B",
      "column_index": 6,
      "raw_value": "34.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "ifbench-unspecified",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.6-27B', row 'Instruction followingIFBench', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Instruction followingIFBench",
      "column_label": "Qwen3.6-27B",
      "column_index": 2,
      "raw_value": "69.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.894,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.6-27B', row 'Document intelligenceOmniDocBench 1.5', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Document intelligenceOmniDocBench 1.5",
      "column_label": "Qwen3.6-27B",
      "column_index": 2,
      "raw_value": "89.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "osworld-verified",
    "score": 0.639,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.6-27B', row 'Computer useOSWorld-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Computer useOSWorld-Verified",
      "column_label": "Qwen3.6-27B",
      "column_index": 2,
      "raw_value": "63.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "terminal-bench-2.1",
    "score": 0.634,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.6-27B', row 'Agentic terminal codingTerminal Bench 2.1 (Terminus)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Agentic terminal codingTerminal Bench 2.1 (Terminus)",
      "column_label": "Qwen3.6-27B",
      "column_index": 2,
      "raw_value": "63.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-6-27b",
    "bench": "webarena-verified",
    "score": 0.488,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.6-27B', row 'Browser useWebArena-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Browser useWebArena-Verified",
      "column_label": "Qwen3.6-27B",
      "column_index": 2,
      "raw_value": "48.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "androidworld",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Mobile useAndroidWorld', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Mobile useAndroidWorld",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "erqa",
    "score": 0.698,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Embodied intelligenceERQA', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Embodied intelligenceERQA",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "69.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "gpqa-diamond",
    "score": 0.903,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Scientific reasoningGPQA Diamond', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Scientific reasoningGPQA Diamond",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "90.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "hle",
    "score": 0.347,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Multidisciplinary reasoningHLE', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Multidisciplinary reasoningHLE",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "34.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "ifbench-unspecified",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Instruction followingIFBench', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Instruction followingIFBench",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "79.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.896,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Competitive codingLiveCodeBench v6', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Competitive codingLiveCodeBench v6",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "89.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "nl2repo",
    "score": 0.411,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Repo-level code generationNL2Repo-Bench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Repo-level code generationNL2Repo-Bench",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "41.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Document intelligenceOmniDocBench 1.5', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Document intelligenceOmniDocBench 1.5",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "91.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "osworld-verified",
    "score": 0.733,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Computer useOSWorld-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Computer useOSWorld-Verified",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "realworldqa",
    "score": 0.869,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Real-world perceptionRealWorldQA', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Real-world perceptionRealWorldQA",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "86.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.576,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Agentic codingSWE-bench Pro', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Agentic codingSWE-bench Pro",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "57.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "terminal-bench-2.1",
    "score": 0.64,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.7-Plus', row 'Agentic terminal codingTerminal Bench 2.1 (Terminus)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Agentic terminal codingTerminal Bench 2.1 (Terminus)",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "64.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-7-plus",
    "bench": "webarena-verified",
    "score": 0.553,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.7-Plus', row 'Browser useWebArena-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Browser useWebArena-Verified",
      "column_label": "Qwen3.7-Plus",
      "column_index": 3,
      "raw_value": "55.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "androidworld",
    "score": 0.819,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Mobile useAndroidWorld', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Mobile useAndroidWorld",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "81.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "erqa",
    "score": 0.655,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Embodied intelligenceERQA', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Embodied intelligenceERQA",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "65.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "ifbench-unspecified",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.8-27B', row 'Instruction followingIFBench', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General",
      "heading": "Text Performance",
      "row_label": "Instruction followingIFBench",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "nl2repo",
    "score": 0.423,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.8-27B', row 'Repo-level code generationNL2Repo-Bench', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Repo-level code generationNL2Repo-Bench",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "42.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.911,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Document intelligenceOmniDocBench 1.5', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Document intelligenceOmniDocBench 1.5",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "91.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "osworld-verified",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Computer useOSWorld-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Computer useOSWorld-Verified",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "realworldqa",
    "score": 0.859,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Real-world perceptionRealWorldQA', section 'General Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Real-world perceptionRealWorldQA",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "85.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.617,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.8-27B', row 'Agentic codingSWE-bench Pro', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Agentic codingSWE-bench Pro",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "61.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "terminal-bench-2.1",
    "score": 0.73,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, Text Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). Qwen3.8 operates in thinking mode by default. Footnotes: SWE-bench Pro evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context, problematic tasks corrected (refined benchmark); NL2Repo-Bench Claude Code harness with repository-access commands disabled; HLE judged by GPT-4o. Column 'Qwen3.8-27B', row 'Agentic terminal codingTerminal Bench 2.1 (Terminus)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Text Performance",
      "row_label": "Agentic terminal codingTerminal Bench 2.1 (Terminus)",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-8-27b",
    "bench": "webarena-verified",
    "score": 0.648,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "source_id": "card-qwen--qwen3.8-27b",
    "by": "lab",
    "date": "2026-08-14",
    "mode": "thinking",
    "note": "Qwen own evaluation in the pinned Qwen3.8-27B model card, VL Performance table (Qwen3.6-27B and Qwen3.7-Plus columns are same-lab evaluations). WebArena-Verified uses the official grader under the OSWorld scaffold. Column 'Qwen3.8-27B', row 'Browser useWebArena-Verified', section 'Agentic Multimodal Intelligence'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic Multimodal Intelligence",
      "heading": "VL Performance",
      "row_label": "Browser useWebArena-Verified",
      "column_label": "Qwen3.8-27B",
      "column_index": 1,
      "raw_value": "64.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "browsecomp",
    "score": 0.68,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5.1', row 'BrowseComp'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp",
      "column_label": "GLM-5.1",
      "column_index": 1,
      "raw_value": "68.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "browsecomp-context-managed",
    "score": 0.793,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5.1', row 'BrowseComp (w/ Context Manage)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp (w/ Context Manage)",
      "column_label": "GLM-5.1",
      "column_index": 1,
      "raw_value": "79.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "cybergym",
    "score": 0.687,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5.1', row 'CyberGym'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "CyberGym",
      "column_label": "GLM-5.1",
      "column_index": 1,
      "raw_value": "68.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "tau3-bench",
    "score": 0.706,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5.1', row 'τ³-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "τ³-Bench",
      "column_label": "GLM-5.1",
      "column_index": 1,
      "raw_value": "70.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "aime-2026",
    "score": 0.954,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'AIME 2026'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "AIME 2026",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "95.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "browsecomp",
    "score": 0.62,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'BrowseComp'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "browsecomp-context-managed",
    "score": 0.759,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'BrowseComp (w/ Context Manage)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp (w/ Context Manage)",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "75.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "cybergym",
    "score": 0.483,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'CyberGym'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "CyberGym",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "48.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "hmmt-feb-2026",
    "score": 0.828,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'HMMT Feb. 2026'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "HMMT Feb. 2026",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "mcp-atlas",
    "score": 0.692,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'MCP-Atlas (Public Set)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "MCP-Atlas (Public Set)",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "69.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "nl2repo",
    "score": 0.359,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'NL2Repo'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "NL2Repo",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "35.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.551,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'SWE-Bench Pro'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "SWE-Bench Pro",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "55.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "tau3-bench",
    "score": 0.692,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'τ³-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "τ³-Bench",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "69.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "terminal-bench-2",
    "score": 0.562,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'Terminal-Bench 2.0 (Terminus-2)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "Terminal-Bench 2.0 (Terminus-2)",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "56.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "toolathlon",
    "score": 0.38,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "source_id": "card-zai-org--glm-5.1",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card Benchmark table (GLM-5 column is a same-lab evaluation). This card has no footnotes; settings as in the GLM-5 series cards are not assumed. Terminal-Bench 2.0 row is the Terminus-2 harness. Column 'GLM-5', row 'Tool-Decathlon'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "Tool-Decathlon",
      "column_label": "GLM-5",
      "column_index": 2,
      "raw_value": "38.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "critpt",
    "score": 0.046,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'CritPt', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "CritPt",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "4.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "hle-with-tools",
    "score": 0.523,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'HLE (w/ Tools)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HLE (w/ Tools)",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "52.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "hmmt-feb-2026",
    "score": 0.826,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'HMMT Feb. 2026', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HMMT Feb. 2026",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "82.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "hmmt-nov-2025",
    "score": 0.94,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'HMMT Nov. 2025', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HMMT Nov. 2025",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "94",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "imo-answerbench",
    "score": 0.838,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'IMOAnswerBench', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "IMOAnswerBench",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "83.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "mcp-atlas",
    "score": 0.718,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'MCP-Atlas (Public Set)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic",
      "heading": "Benchmark",
      "row_label": "MCP-Atlas (Public Set)",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "71.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "nl2repo",
    "score": 0.427,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'NL2Repo', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Benchmark",
      "row_label": "NL2Repo",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "42.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "terminal-bench-2.1",
    "score": 0.635,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'Terminal Bench 2.1 (Terminus-2)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Benchmark",
      "row_label": "Terminal Bench 2.1 (Terminus-2)",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "63.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-1",
    "bench": "toolathlon",
    "score": 0.407,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.1', row 'Tool-Decathlon', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic",
      "heading": "Benchmark",
      "row_label": "Tool-Decathlon",
      "column_label": "GLM-5.1",
      "column_index": 2,
      "raw_value": "40.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "critpt",
    "score": 0.209,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'CritPt', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "CritPt",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "20.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "hle-with-tools",
    "score": 0.547,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'HLE (w/ Tools)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HLE (w/ Tools)",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "54.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "hmmt-feb-2026",
    "score": 0.925,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'HMMT Feb. 2026', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HMMT Feb. 2026",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "92.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "hmmt-nov-2025",
    "score": 0.944,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'HMMT Nov. 2025', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "HMMT Nov. 2025",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "94.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "imo-answerbench",
    "score": 0.91,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'IMOAnswerBench', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning",
      "heading": "Benchmark",
      "row_label": "IMOAnswerBench",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "91.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "mcp-atlas",
    "score": 0.768,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'MCP-Atlas (Public Set)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic",
      "heading": "Benchmark",
      "row_label": "MCP-Atlas (Public Set)",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "nl2repo",
    "score": 0.489,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'NL2Repo', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Benchmark",
      "row_label": "NL2Repo",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "48.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "terminal-bench-2.1",
    "score": 0.81,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'Terminal Bench 2.1 (Terminus-2)', section 'Coding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Coding",
      "heading": "Benchmark",
      "row_label": "Terminal Bench 2.1 (Terminus-2)",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5-2",
    "bench": "toolathlon",
    "score": 0.482,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "source_id": "card-zai-org--glm-5.2",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5.2 model card Benchmark table (GLM-5.1 column is a same-lab evaluation). GLM-5.2 card footnotes: temperature=1.0, top_p=0.95, max generation 163,840, HLE text-only subset by default; SWE-Bench Pro with OpenHands (temperature=1, top_p=1, 32k new tokens, 400K context); NL2Repo temperature=1.0, 48k new tokens, 400k context with anti-hacking judgement; Terminal-Bench 2.1 (Terminus-2) timeout 4h, 256K context; MCP-Atlas 500-task public subset, Gemini-3.0-Pro judge; Tool-Decathlon official service, max_token 128K. Third-party-run rows (FrontierSWE, PostTrainBench, SWE-Marathon) and DeepSWE/ProgramBench are not used. Column 'GLM-5.2', row 'Tool-Decathlon', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic",
      "heading": "Benchmark",
      "row_label": "Tool-Decathlon",
      "column_label": "GLM-5.2",
      "column_index": 1,
      "raw_value": "48.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "math-subset-unspecified",
    "score": 0.962,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "source_id": "card2-allenai-olmo-3-1-32b-think",
    "by": "lab",
    "date": "2026-01-05",
    "mode": "thinking",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card, Evaluation table, 'Olmo 3.1 32B Think' column. Only rows not already recorded from the Olmo 3 report are used; MATH subset not stated. Column 'Olmo 3.1 32B Think', row 'MATH', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "MATH",
      "column_label": "Olmo 3.1 32B Think",
      "column_index": 1,
      "raw_value": "96.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "omega",
    "score": 0.534,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "source_id": "card2-allenai-olmo-3-1-32b-think",
    "by": "lab",
    "date": "2026-01-05",
    "mode": "thinking",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card, Evaluation table, 'Olmo 3.1 32B Think' column. Only rows not already recorded from the Olmo 3 report are used; MATH subset not stated. Column 'Olmo 3.1 32B Think', row 'OMEGA', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "OMEGA",
      "column_label": "Olmo 3.1 32B Think",
      "column_index": 1,
      "raw_value": "53.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "bfcl-version-unspecified",
    "score": 0.498,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "source_id": "card2-allenai-olmo-3-7b-instruct",
    "by": "lab",
    "date": "2026-06-25",
    "mode": "instruct",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Instruct model card, Evaluation table, final 'Olmo3 Instruct 7B' column (not SFT/DPO). Only rows not already recorded from the Olmo 3 report are used; MATH subset and BFCL version not stated (BFCL listed under Tool Use). Column 'Olmo3 Instruct 7B', row 'BFCL', section 'Tool Use'. (Benchmark id harmonised from bfcl-unversioned.)",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Tool Use",
      "heading": "Evaluation",
      "row_label": "BFCL",
      "column_label": "Olmo3 Instruct 7B",
      "column_index": 4,
      "raw_value": "49.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "math-subset-unspecified",
    "score": 0.873,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "source_id": "card2-allenai-olmo-3-7b-instruct",
    "by": "lab",
    "date": "2026-06-25",
    "mode": "instruct",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Instruct model card, Evaluation table, final 'Olmo3 Instruct 7B' column (not SFT/DPO). Only rows not already recorded from the Olmo 3 report are used; MATH subset and BFCL version not stated (BFCL listed under Tool Use). Column 'Olmo3 Instruct 7B', row 'MATH', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "MATH",
      "column_label": "Olmo3 Instruct 7B",
      "column_index": 4,
      "raw_value": "87.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "omega",
    "score": 0.289,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "source_id": "card2-allenai-olmo-3-7b-instruct",
    "by": "lab",
    "date": "2026-06-25",
    "mode": "instruct",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Instruct model card, Evaluation table, final 'Olmo3 Instruct 7B' column (not SFT/DPO). Only rows not already recorded from the Olmo 3 report are used; MATH subset and BFCL version not stated (BFCL listed under Tool Use). Column 'Olmo3 Instruct 7B', row 'OMEGA', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "OMEGA",
      "column_label": "Olmo3 Instruct 7B",
      "column_index": 4,
      "raw_value": "28.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "math-subset-unspecified",
    "score": 0.951,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "source_id": "card2-allenai-olmo-3-7b-think",
    "by": "lab",
    "date": "2026-06-25",
    "mode": "thinking",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Think model card, Evaluation table, final 'Olmo 3 Think 7B' column (not the SFT/DPO intermediate columns). Only rows not already recorded from the Olmo 3 report are used; MATH subset not stated. Column 'Olmo 3 Think 7B', row 'MATH', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "MATH",
      "column_label": "Olmo 3 Think 7B",
      "column_index": 4,
      "raw_value": "95.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "omega",
    "score": 0.378,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "source_id": "card2-allenai-olmo-3-7b-think",
    "by": "lab",
    "date": "2026-06-25",
    "mode": "thinking",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Think model card, Evaluation table, final 'Olmo 3 Think 7B' column (not the SFT/DPO intermediate columns). Only rows not already recorded from the Olmo 3 report are used; MATH subset not stated. Column 'Olmo 3 Think 7B', row 'OMEGA', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "OMEGA",
      "column_label": "Olmo 3 Think 7B",
      "column_index": 4,
      "raw_value": "37.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "livecodebench-unversioned",
    "score": 0.492,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/blob/e9b33add76883f293d6bf61f6bd89b497e80e335/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-0324",
    "by": "lab",
    "date": "2025-03-27",
    "mode": "non-thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3-0324 model card, 'Reasoning Capabilities' bullet list (predecessor → V3-0324). Bullet 'LiveCodeBench: 39.2 → 49.2'; version/window not stated.",
    "evidence": {
      "kind": "text",
      "quote": "LiveCodeBench: 39.2 → 49.2 (+10.0)",
      "row_label": "LiveCodeBench",
      "column_label": "DeepSeek-V3-0324 (after arrow)",
      "raw_value": "49.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3",
    "bench": "livecodebench-unversioned",
    "score": 0.392,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/blob/e9b33add76883f293d6bf61f6bd89b497e80e335/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-0324",
    "by": "lab",
    "date": "2025-03-27",
    "mode": "non-thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3-0324 model card, 'Reasoning Capabilities' bullet list (predecessor → V3-0324). Bullet 'LiveCodeBench: 39.2 → 49.2'; version/window not stated.",
    "evidence": {
      "kind": "text",
      "quote": "LiveCodeBench: 39.2 → 49.2 (+10.0)",
      "row_label": "LiveCodeBench",
      "column_label": "DeepSeek-V3 (predecessor, before arrow)",
      "raw_value": "39.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aider-polyglot",
    "score": 0.716,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'Aider-Polyglot (Acc.)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "Aider-Polyglot (Acc.)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "71.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2024",
    "score": 0.914,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'AIME 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "AIME 2024 (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "91.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "browsecomp",
    "score": 0.089,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'BrowseComp', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "BrowseComp",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "8.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "browsecomp-zh",
    "score": 0.357,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'BrowseComp_zh', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "BrowseComp_zh",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "35.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hle-with-tools",
    "score": 0.248,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'Humanity's Last Exam (Python + Search)', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "Humanity's Last Exam (Python + Search)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "24.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.794,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'HMMT 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "HMMT 2025 (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "79.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.733,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'LiveCodeBench (2408-2505) (Pass@1)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "LiveCodeBench (2408-2505) (Pass@1)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "mmlu-redux",
    "score": 0.934,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'MMLU-Redux (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "MMLU-Redux (EM)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "93.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "swe-bench-multilingual",
    "score": 0.305,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'SWE-bench Multilingual (Agent mode)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "SWE-bench Multilingual (Agent mode)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "30.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "terminal-bench-1-terminus1",
    "score": 0.057,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek R1 0528', row 'Terminal-bench (Terminus 1 framework)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "Terminal-bench (Terminus 1 framework)",
      "column_label": "DeepSeek R1 0528",
      "column_index": 5,
      "raw_value": "5.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "aider-polyglot",
    "score": 0.551,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'Aider-Polyglot (Acc.)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "Aider-Polyglot (Acc.)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "55.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "aime-2024",
    "score": 0.594,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'AIME 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "AIME 2024 (Pass@1)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "59.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "aime-2025",
    "score": 0.513,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'AIME 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "AIME 2025 (Pass@1)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "51.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "gpqa-diamond",
    "score": 0.684,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'GPQA-Diamond (Pass@1)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "GPQA-Diamond (Pass@1)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "68.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.292,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'HMMT 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "HMMT 2025 (Pass@1)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "29.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.43,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'LiveCodeBench (2408-2505) (Pass@1)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "LiveCodeBench (2408-2505) (Pass@1)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "43.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "mmlu-pro",
    "score": 0.812,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'MMLU-Pro (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "MMLU-Pro (EM)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "81.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "mmlu-redux",
    "score": 0.905,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'MMLU-Redux (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "MMLU-Redux (EM)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "90.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "swe-bench-multilingual",
    "score": 0.293,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'SWE-bench Multilingual (Agent mode)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "SWE-bench Multilingual (Agent mode)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "29.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "swe-bench-verified",
    "score": 0.454,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'SWE Verified (Agent mode)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "SWE Verified (Agent mode)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "45.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "terminal-bench-1-terminus1",
    "score": 0.133,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3 0324', row 'Terminal-bench (Terminus 1 framework)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "Terminal-bench (Terminus 1 framework)",
      "column_label": "DeepSeek V3 0324",
      "column_index": 3,
      "raw_value": "13.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aider-polyglot",
    "score": 0.763,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'Aider-Polyglot (Acc.)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "Aider-Polyglot (Acc.)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "76.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aime-2024",
    "score": 0.931,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'AIME 2024 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "AIME 2024 (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "93.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aime-2025",
    "score": 0.884,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'AIME 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "AIME 2025 (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "88.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "browsecomp",
    "score": 0.3,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'BrowseComp', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "BrowseComp",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "browsecomp-zh",
    "score": 0.492,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'BrowseComp_zh', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "BrowseComp_zh",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "49.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "gpqa-diamond",
    "score": 0.801,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'GPQA-Diamond (Pass@1)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "GPQA-Diamond (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "80.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "hle",
    "score": 0.159,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'Humanity's Last Exam (Pass@1)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "Humanity's Last Exam (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "15.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "hle-with-tools",
    "score": 0.298,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'Humanity's Last Exam (Python + Search)', section 'Search Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Search Agent",
      "heading": "Evaluation",
      "row_label": "Humanity's Last Exam (Python + Search)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "29.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.842,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'HMMT 2025 (Pass@1)', section 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math",
      "heading": "Evaluation",
      "row_label": "HMMT 2025 (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "84.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.748,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'LiveCodeBench (2408-2505) (Pass@1)', section 'Code'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code",
      "heading": "Evaluation",
      "row_label": "LiveCodeBench (2408-2505) (Pass@1)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "74.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-pro",
    "score": 0.848,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'MMLU-Pro (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "MMLU-Pro (EM)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "84.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-redux",
    "score": 0.937,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (V3.1 thinking-mode column; DeepSeek V3 0324 and R1 0528 columns are same-lab evaluations). Notes: search agents use DeepSeek's internal search framework (commercial search API + webpage filter + 128K context; R1-0528 search results use a pre-defined workflow); SWE-bench uses the internal code agent framework; HLE text-only subset. Column 'DeepSeek V3.1-Thinking', row 'MMLU-Redux (EM)', section 'General'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "General",
      "heading": "Evaluation",
      "row_label": "MMLU-Redux (EM)",
      "column_label": "DeepSeek V3.1-Thinking",
      "column_index": 4,
      "raw_value": "93.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "swe-bench-multilingual",
    "score": 0.545,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "non-thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table, V3.1-NonThinking column; used only for rows with no thinking-mode value (code-agent rows are reported in non-thinking agent mode only). SWE-bench uses DeepSeek's internal code agent framework; Terminal-bench uses the Terminus 1 framework. Column 'DeepSeek V3.1-NonThinking', row 'SWE-bench Multilingual (Agent mode)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "SWE-bench Multilingual (Agent mode)",
      "column_label": "DeepSeek V3.1-NonThinking",
      "column_index": 2,
      "raw_value": "54.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "swe-bench-verified",
    "score": 0.66,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "non-thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table, V3.1-NonThinking column; used only for rows with no thinking-mode value (code-agent rows are reported in non-thinking agent mode only). SWE-bench uses DeepSeek's internal code agent framework; Terminal-bench uses the Terminus 1 framework. Column 'DeepSeek V3.1-NonThinking', row 'SWE Verified (Agent mode)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "SWE Verified (Agent mode)",
      "column_label": "DeepSeek V3.1-NonThinking",
      "column_index": 2,
      "raw_value": "66.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-1",
    "bench": "terminal-bench-1-terminus1",
    "score": 0.313,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-1",
    "by": "lab",
    "date": "2025-09-05",
    "mode": "non-thinking",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table, V3.1-NonThinking column; used only for rows with no thinking-mode value (code-agent rows are reported in non-thinking agent mode only). SWE-bench uses DeepSeek's internal code agent framework; Terminal-bench uses the Terminus 1 framework. Column 'DeepSeek V3.1-NonThinking', row 'Terminal-bench (Terminus 1 framework)', section 'Code Agent'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Code Agent",
      "heading": "Evaluation",
      "row_label": "Terminal-bench (Terminus 1 framework)",
      "column_label": "DeepSeek V3.1-NonThinking",
      "column_index": 2,
      "raw_value": "31.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "aime-2025",
    "score": 0.931,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '93.1' for 'AIME 2025 (Pass@1)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "AIME 2025 (Pass@1)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "93.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "hle",
    "score": 0.251,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '25.1' for 'HLE (Pass@1)'. Text-only vs full set not stated in this chart.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "HLE (Pass@1)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "25.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.902,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '90.2' for 'HMMT 2025 (Pass@1)'. HMMT edition not stated.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "HMMT 2025 (Pass@1)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "90.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "swe-bench-verified",
    "score": 0.731,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '73.1' for 'SWE Verified (Resolved)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "SWE Verified (Resolved)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "73.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "tau2-bench",
    "score": 0.803,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '80.3' for 'τ² Bench (Pass@1)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "τ² Bench (Pass@1)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "80.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "terminal-bench-2",
    "score": 0.464,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '46.4' for 'Terminal Bench 2.0 (Acc)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "Terminal Bench 2.0 (Acc)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "46.4",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v3-2",
    "bench": "toolathlon",
    "score": 0.352,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-2",
    "by": "lab",
    "date": "2025-12-01",
    "mode": "thinking",
    "note": "DeepSeek own results in the benchmark chart committed in the pinned DeepSeek-V3.2 repository (assets/benchmark.png, embedded in the README; visually read; archived copy hashed in evidence). DeepSeek-V3.2-Thinking bars (hatched); the separate V3.2-Speciale variant and competitor bars are not used; Codeforces rating not used. Printed label '35.2' for 'Tool Decathlon (Pass@1)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.png",
      "row_label": "Tool Decathlon (Pass@1)",
      "column_label": "DeepSeek-V3.2-Thinking",
      "raw_value": "35.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/benchmark.png",
      "asset_sha256": "08a78f9d224d1748010d5d8da403f28efa4c36bcd1d6a33dc7515b50e256ffd9",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3.2__benchmark.png"
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "agents-last-exam",
    "score": 0.318,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'Agent's Last Exam (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Agent's Last Exam (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "31.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "automationbench",
    "score": 0.548,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'AutomationBench (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "AutomationBench (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "54.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "babyvision-with-tools",
    "score": 0.896,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'BabyVision w/ tools (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "BabyVision w/ tools (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "89.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "chartography-with-tools",
    "score": 0.789,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'Chartography w/ tools (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Chartography w/ tools (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "cybergym",
    "score": 0.881,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'CyberGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "CyberGym (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "88.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "deepswe-v1-1",
    "score": 0.742,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'DeepSWE v1.1 (Resolved)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "DeepSWE v1.1 (Resolved)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "74.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "exploitgym",
    "score": 0.153,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'ExploitGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ExploitGym (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "15.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "gpqa-diamond",
    "score": 0.909,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'GPQA Diamond (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "GPQA Diamond (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "90.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "hle",
    "score": 0.391,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Cell '36.8 (39.1†)': the † value 39.1 is the HLE text-only subset (footnote), used here for consistency with the text-only DS-V4 cells; 36.8 (full set) not recorded.",
    "evidence": {
      "kind": "table-cell-manual",
      "quote": "| 36.8 (39.1†) |",
      "row_label": "HLE (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "raw_value": "39.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "hle-with-tools",
    "score": 0.639,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'HLE w/ tools (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "HLE w/ tools (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "63.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "matharena-apex",
    "score": 0.656,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'MathArena Apex (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "MathArena Apex (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "65.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "nl2repo",
    "score": 0.64,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'NL2Repo-Bench (Score)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "NL2Repo-Bench (Score)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "64.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "programbench-almost1",
    "score": 0.203,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'ProgramBench (Almost@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ProgramBench (Almost@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "20.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "sec-bench-pro",
    "score": 0.628,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'SEC-Bench Pro (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "SEC-Bench Pro (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "62.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.906,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'Terminal-Bench 2.1 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 2.1 (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "90.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "terminal-bench-3",
    "score": 0.3,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'Terminal-Bench 3.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 3.0 (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "terminal-bench-4",
    "score": 0.312,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'Terminal-Bench 4.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 4.0 (Pass@1)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "31.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "zerobench-main-with-tools-pass5",
    "score": 0.49,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4.1-Flash', row 'ZeroBench-main w/ tools (Pass@5)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ZeroBench-main w/ tools (Pass@5)",
      "column_label": "DS-V4.1-Flash",
      "column_index": 7,
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "agents-last-exam",
    "score": 0.252,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'Agent's Last Exam (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Agent's Last Exam (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "25.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "automationbench",
    "score": 0.377,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'AutomationBench (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "AutomationBench (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "37.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "cybergym",
    "score": 0.767,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'CyberGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "CyberGym (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "76.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "deepswe-v1-1",
    "score": 0.544,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'DeepSWE v1.1 (Resolved)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "DeepSWE v1.1 (Resolved)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "54.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "exploitgym",
    "score": 0.018,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'ExploitGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ExploitGym (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "1.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "gpqa-diamond",
    "score": 0.899,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'GPQA Diamond (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "GPQA Diamond (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "89.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "hle",
    "score": 0.378,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'HLE (Pass@1)', section 'Reasoning'. Marker '†' in cell: text-only subset of HLE (footnote).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "HLE (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "37.8†",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "hle-with-tools",
    "score": 0.515,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'HLE w/ tools (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "HLE w/ tools (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "51.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "matharena-apex",
    "score": 0.586,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'MathArena Apex (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "MathArena Apex (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "58.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "nl2repo",
    "score": 0.542,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'NL2Repo-Bench (Score)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "NL2Repo-Bench (Score)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "54.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "sec-bench-pro",
    "score": 0.309,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'SEC-Bench Pro (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "SEC-Bench Pro (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "30.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.827,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'Terminal-Bench 2.1 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 2.1 (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "82.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "terminal-bench-3",
    "score": 0.076,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'Terminal-Bench 3.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 3.0 (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "7.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "terminal-bench-4",
    "score": 0.07,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Flash', row 'Terminal-Bench 4.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 4.0 (Pass@1)",
      "column_label": "DS-V4-Flash",
      "column_index": 6,
      "raw_value": "7.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "agents-last-exam",
    "score": 0.257,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'Agent's Last Exam (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Agent's Last Exam (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "25.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "automationbench",
    "score": 0.432,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'AutomationBench (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "AutomationBench (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "43.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "cybergym",
    "score": 0.833,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'CyberGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "CyberGym (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "83.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "deepswe-v1-1",
    "score": 0.627,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'DeepSWE v1.1 (Resolved)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "DeepSWE v1.1 (Resolved)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "62.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "exploitgym",
    "score": 0.054,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'ExploitGym (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ExploitGym (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "5.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "gpqa-diamond",
    "score": 0.924,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'GPQA Diamond (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "GPQA Diamond (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "92.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "hle",
    "score": 0.427,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'HLE (Pass@1)', section 'Reasoning'. Marker '†' in cell: text-only subset of HLE (footnote).",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "HLE (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "42.7†",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "hle-with-tools",
    "score": 0.6,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'HLE w/ tools (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "HLE w/ tools (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "60.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "matharena-apex",
    "score": 0.653,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'MathArena Apex (Pass@1)', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Reasoning",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "MathArena Apex (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "65.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "nl2repo",
    "score": 0.615,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'NL2Repo-Bench (Score)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "NL2Repo-Bench (Score)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "61.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "programbench-almost1",
    "score": 0.155,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'ProgramBench (Almost@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "ProgramBench (Almost@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "15.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "sec-bench-pro",
    "score": 0.564,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'SEC-Bench Pro (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "SEC-Bench Pro (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "56.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "terminal-bench-2.1",
    "score": 0.879,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'Terminal-Bench 2.1 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 2.1 (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "87.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "terminal-bench-3",
    "score": 0.118,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'Terminal-Bench 3.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 3.0 (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "11.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "terminal-bench-4",
    "score": 0.124,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-1-flash",
    "by": "lab",
    "date": "2026-09-10",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4.1-Flash model card, 'Comparison with frontier models (Max reasoning effort)' table. All instruct results use maximum effort (reasoning_effort=100), temperature=1.0, top_p=0.95. Code-agent benchmarks (Terminal-Bench 2.1/3.0/4.0, DeepSWE v1.1, NL2Repo-Bench, ProgramBench) use the Minimal mode of DeepSeek Harness with a 1M-token context; DeepSWE v1.1 uses the mini-SWE harness and SEC-Bench Pro the Claude Code harness; visual agent benchmarks (Chartography, BabyVision, ZeroBench) use the Claude Code harness with 512k context; Agent's Last Exam and AutomationBench use official scaffolds. † = HLE text-only subset. Columns 'DS-V4-Pro' and 'DS-V4-Flash' are mapped to V4-Pro-0813 and V4-Flash-0731: their values are identical to the dated 0813/0731 cards on Terminal-Bench 2.1, NL2Repo, CyberGym, DeepSWE, HLE and HLE w/ tools. Column 'DS-V4-Pro', row 'Terminal-Bench 4.0 (Pass@1)', section 'Agentic'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Agentic",
      "heading": "Comparison with frontier models (Max reasoning effort)",
      "row_label": "Terminal-Bench 4.0 (Pass@1)",
      "column_label": "DS-V4-Pro",
      "column_index": 5,
      "raw_value": "12.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "automationbench-public",
    "score": 0.251,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813/blob/72e1d3230f6c080a530b0a1d46f8eb4602340597/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-pro-0813",
    "by": "lab",
    "date": "2026-08-13",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4-Pro-0813 model card (Introduction table). Code-agent tasks use the minimal mode of DeepSeek Harness, max reasoning effort, temperature=1.0, top_p=0.95. DSBench rows are internal († ) and not used; HLE dual cell not used here (taken from the V4.1-Flash card). Column 'DeepSeek-V4-Flash-0731', row 'AutomationBench (Public)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "AutomationBench (Public)",
      "column_label": "DeepSeek-V4-Flash-0731",
      "column_index": 2,
      "raw_value": "25.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "toolathlon-verified",
    "score": 0.703,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813/blob/72e1d3230f6c080a530b0a1d46f8eb4602340597/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-pro-0813",
    "by": "lab",
    "date": "2026-08-13",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4-Pro-0813 model card (Introduction table). Code-agent tasks use the minimal mode of DeepSeek Harness, max reasoning effort, temperature=1.0, top_p=0.95. DSBench rows are internal († ) and not used; HLE dual cell not used here (taken from the V4.1-Flash card). Column 'DeepSeek-V4-Flash-0731', row 'Toolathlon-Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "Toolathlon-Verified",
      "column_label": "DeepSeek-V4-Flash-0731",
      "column_index": 2,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "automationbench-public",
    "score": 0.318,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813/blob/72e1d3230f6c080a530b0a1d46f8eb4602340597/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-pro-0813",
    "by": "lab",
    "date": "2026-08-13",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4-Pro-0813 model card (Introduction table). Code-agent tasks use the minimal mode of DeepSeek Harness, max reasoning effort, temperature=1.0, top_p=0.95. DSBench rows are internal († ) and not used; HLE dual cell not used here (taken from the V4.1-Flash card). Column 'DeepSeek-V4-Pro-0813', row 'AutomationBench (Public)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "AutomationBench (Public)",
      "column_label": "DeepSeek-V4-Pro-0813",
      "column_index": 1,
      "raw_value": "31.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "toolathlon-verified",
    "score": 0.741,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813/blob/72e1d3230f6c080a530b0a1d46f8eb4602340597/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v4-pro-0813",
    "by": "lab",
    "date": "2026-08-13",
    "mode": "max reasoning effort",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V4-Pro-0813 model card (Introduction table). Code-agent tasks use the minimal mode of DeepSeek Harness, max reasoning effort, temperature=1.0, top_p=0.95. DSBench rows are internal († ) and not used; HLE dual cell not used here (taken from the V4.1-Flash card). Column 'DeepSeek-V4-Pro-0813', row 'Toolathlon-Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "Toolathlon-Verified",
      "column_label": "DeepSeek-V4-Pro-0813",
      "column_index": 1,
      "raw_value": "74.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "ami-wer",
    "score": 0.1463,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "AMI",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "14.63",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "earnings21-longform-wer",
    "score": 0.106,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used. Long-form: sequential decoding, condition_on_prev_tokens=False, return_timestamps=True.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long-Form Evaluation",
      "row_label": "earnings21",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "10.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "earnings22-longform-wer",
    "score": 0.1419,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used. Long-form: sequential decoding, condition_on_prev_tokens=False, return_timestamps=True.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long-Form Evaluation",
      "row_label": "earnings22",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "14.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "earnings22-wer",
    "score": 0.1129,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "Earnings22",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "11.29",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "gigaspeech-wer",
    "score": 0.0984,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "Gigaspeech",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "9.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0237,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "LS Clean",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "2.37",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "librispeech-test-other-wer",
    "score": 0.0504,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "LS Other",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "5.04",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "meanwhile-longform-wer",
    "score": 0.0679,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used. Long-form: sequential decoding, condition_on_prev_tokens=False, return_timestamps=True.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long-Form Evaluation",
      "row_label": "meanwhile",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "6.79",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "rev16-longform-wer",
    "score": 0.1398,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used. Long-form: sequential decoding, condition_on_prev_tokens=False, return_timestamps=True.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long-Form Evaluation",
      "row_label": "rev16",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "13.98",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "spgispeech-wer",
    "score": 0.0287,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "SPGISpeech",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "2.87",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "tedlium-longform-wer",
    "score": 0.0463,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used. Long-form: sequential decoding, condition_on_prev_tokens=False, return_timestamps=True.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long-Form Evaluation",
      "row_label": "tedlium-long-form",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "4.63",
      "scale_divisor": 100
    }
  },
  {
    "model": "distil-large-v3-5",
    "bench": "tedlium-wer",
    "score": 0.0364,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "source_id": "card2-distil-whisper-distil-large-v3-5",
    "by": "lab",
    "date": "2026-04-13",
    "mode": "greedy-default",
    "note": "Hugging Face Distil-Whisper team's own evaluation of distil-large-v3.5 (column 'distil-v3.5'); WER after the Whisper basic normaliser linked in the card (lowercasing, symbol/punctuation removal). Converted from percent. Whisper large-v3/turbo columns (other lab's models) not used.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Short-Form Evaluation",
      "row_label": "Tedlium",
      "column_label": "distil-v3.5",
      "column_index": 5,
      "raw_value": "3.64",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "tau2-bench",
    "score": 0.162,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "non-thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 3 27B (no think)', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 3 27B (no think)",
      "column_index": 6,
      "raw_value": "16.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-12b",
    "bench": "tau2-bench",
    "score": 0.69,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 12B Unified', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 4 12B Unified",
      "column_index": 3,
      "raw_value": "69.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "hle-with-tools",
    "score": 0.172,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 26B A4B', row 'HLE with search'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "HLE with search",
      "column_label": "Gemma 4 26B A4B",
      "column_index": 2,
      "raw_value": "17.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "tau2-bench",
    "score": 0.682,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 26B A4B', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 4 26B A4B",
      "column_index": 2,
      "raw_value": "68.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-31b",
    "bench": "hle-with-tools",
    "score": 0.265,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 31B', row 'HLE with search'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "HLE with search",
      "column_label": "Gemma 4 31B",
      "column_index": 1,
      "raw_value": "26.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-31b",
    "bench": "tau2-bench",
    "score": 0.769,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 31B', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 4 31B",
      "column_index": 1,
      "raw_value": "76.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau2-bench",
    "score": 0.245,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 E2B', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 4 E2B",
      "column_index": 5,
      "raw_value": "24.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau2-bench",
    "score": 0.422,
    "source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
    "source_id": "card2-google-gemma-4-31b-it",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "thinking",
    "note": "Google own evaluation in the pinned google/gemma-4-31B-it model card (Apache-2.0), Benchmark Results table (all Gemma columns are Google evaluations; Gemma 4 thinking mode, Gemma 3 27B without thinking). Cells already recorded from the Gemma 4 technical report are not repeated; audio rows and Codeforces ELO not used. Tau2 is the average over the three domains; HLE with search uses tool access. Column 'Gemma 4 E4B', row 'Tau2 (average over 3)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "**Benchmark Results**",
      "heading": "**Benchmark Results**",
      "row_label": "Tau2 (average over 3)",
      "column_label": "Gemma 4 E4B",
      "column_index": 4,
      "raw_value": "42.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "aime-2025",
    "score": 0.367,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'AIME 2025', section 'High school math competition'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "High school math competition",
      "heading": "Extended Thinking",
      "row_label": "AIME 2025",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "36.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "bfcl-version-unspecified",
    "score": 0.888,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'BFCL', section 'Tool Calling'. (Benchmark id harmonised from bfcl-unversioned.)",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Tool Calling",
      "heading": "Extended Thinking",
      "row_label": "BFCL",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "88.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "global-mmlu",
    "score": 0.641,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'Global MMLU', section 'Multilingual Q&A'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Multilingual Q&A",
      "heading": "Extended Thinking",
      "row_label": "Global MMLU",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "64.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "gpqa-diamond",
    "score": 0.417,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'GPQA Diamond', section 'Graduate-level reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Graduate-level reasoning",
      "heading": "Extended Thinking",
      "row_label": "GPQA Diamond",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "41.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "gsm-plus",
    "score": 0.834,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'GSM-Plus', section 'Math problem-solving'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Math problem-solving",
      "heading": "Extended Thinking",
      "row_label": "GSM-Plus",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "ifeval-unspecified",
    "score": 0.712,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'IFEval', section 'Instruction following'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Instruction following",
      "heading": "Extended Thinking",
      "row_label": "IFEval",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "71.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "livecodebench-v4",
    "score": 0.3,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'LiveCodeBench v4', section 'Competitive programming'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Competitive programming",
      "heading": "Extended Thinking",
      "row_label": "LiveCodeBench v4",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "smollm3-3b",
    "bench": "mixeval-hard",
    "score": 0.308,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "source_id": "card2-huggingfacetb-smollm3-3b",
    "by": "lab",
    "date": "2025-09-10",
    "mode": "extended thinking",
    "note": "Hugging Face own evaluation in the pinned SmolLM3-3B model card, 'Extended Thinking' table (reasoning mode; the No Extended Thinking table is not used for this hybrid model). All evaluations zero-shot unless stated, run with lighteval. BFCL version and IFEval aggregation not stated. Column 'SmoLLM3-3B', row 'MixEval Hard', section 'Alignment'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Alignment",
      "heading": "Extended Thinking",
      "row_label": "MixEval Hard",
      "column_label": "SmoLLM3-3B",
      "column_index": 2,
      "raw_value": "30.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "bbh",
    "score": 0.8162,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'BBH' (3-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "BBH",
      "metric_label": "3-shot, CoT",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "81.62",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "bfcl-v3",
    "score": 0.6469,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'BFCL v3', section 'Tool Calling Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Tool Calling Tasks",
      "heading": "print output",
      "row_label": "BFCL v3",
      "metric_label": "",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "64.69",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "bigcodebench",
    "score": 0.4623,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'BigCodeBench' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "BigCodeBench",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "46.23",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "cruxeval-o",
    "score": 0.5025,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'CRUXEval-O' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "CRUXEval-O",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "50.25",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "deepmind-math",
    "score": 0.5933,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "DeepMind Math",
      "metric_label": "0-shot, CoT",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "59.33",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "gpqa-unspecified",
    "score": 0.4063,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'GPQA' (0-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "GPQA",
      "metric_label": "0-shot, CoT",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "40.63",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "gsm-symbolic",
    "score": 0.8738,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'GSM8K Symbolic' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM8K Symbolic",
      "metric_label": "8-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "87.38",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "gsm8k",
    "score": 0.8727,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'GSM8K' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM8K",
      "metric_label": "8-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "87.27",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "humaneval",
    "score": 0.88,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'HumanEval' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "88",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "humaneval-plus",
    "score": 0.83,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'HumanEval+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval+",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "83",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "ifeval-instruction-strict",
    "score": 0.8987,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment Tasks",
      "heading": "print output",
      "row_label": "IFEval",
      "metric_label": "Instruct, Strict",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "89.87",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "ifeval-prompt-strict",
    "score": 0.8522,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment Tasks",
      "heading": "print output",
      "row_label": "IFEval",
      "metric_label": "Prompt, Strict",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "85.22",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "include",
    "score": 0.6397,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'INCLUDE' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "INCLUDE",
      "metric_label": "5-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "63.97",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mbpp",
    "score": 0.84,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MBPP' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "84",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mbpp-plus",
    "score": 0.71,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MBPP+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP+",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "71",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mgsm-5lang",
    "score": 0.3872,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MGSM' (8-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MGSM",
      "metric_label": "8-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "38.72",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "minerva-math",
    "score": 0.74,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "Minerva Math",
      "metric_label": "0-shot, CoT",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "74",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmlu",
    "score": 0.7844,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MMLU' (5-shot), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU",
      "metric_label": "5-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "78.44",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmlu-pro",
    "score": 0.5547,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU-Pro",
      "metric_label": "5-shot, CoT",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "55.47",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmmlu",
    "score": 0.6969,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MMMLU' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MMMLU",
      "metric_label": "5-shot",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "69.69",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "multipl-e",
    "score": 0.5737,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'H Small MoE', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MULTIPLE",
      "metric_label": "pass@1",
      "column_label": "H Small MoE",
      "column_index": 5,
      "raw_value": "57.37",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "bbh",
    "score": 0.7248,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'BBH' (3-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "BBH",
      "metric_label": "3-shot, CoT",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "72.48",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "bfcl-v3",
    "score": 0.5998,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'BFCL v3', section 'Tool Calling Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Tool Calling Tasks",
      "heading": "print output",
      "row_label": "BFCL v3",
      "metric_label": "",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "59.98",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "bigcodebench",
    "score": 0.3921,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "BigCodeBench",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "39.21",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "cruxeval-o",
    "score": 0.415,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "CRUXEval-O",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "41.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "deepmind-math",
    "score": 0.4456,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "DeepMind Math",
      "metric_label": "0-shot, CoT",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "44.56",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "gpqa-unspecified",
    "score": 0.3014,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "GPQA",
      "metric_label": "0-shot, CoT",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "30.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "gsm-symbolic",
    "score": 0.7982,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'GSM8K Symbolic' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM8K Symbolic",
      "metric_label": "8-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "79.82",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "gsm8k",
    "score": 0.8545,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'GSM8K' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM8K",
      "metric_label": "8-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "85.45",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "humaneval",
    "score": 0.8,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'HumanEval' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "80",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "humaneval-plus",
    "score": 0.72,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'HumanEval+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval+",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "72",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "ifeval-instruction-strict",
    "score": 0.855,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment Tasks",
      "heading": "print output",
      "row_label": "IFEval",
      "metric_label": "Instruct, Strict",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "85.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "ifeval-prompt-strict",
    "score": 0.7912,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment Tasks",
      "heading": "print output",
      "row_label": "IFEval",
      "metric_label": "Prompt, Strict",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "79.12",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "include",
    "score": 0.5162,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "INCLUDE",
      "metric_label": "5-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "51.62",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mbpp",
    "score": 0.72,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MBPP' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "72",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mbpp-plus",
    "score": 0.64,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MBPP+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP+",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "64",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mgsm-5lang",
    "score": 0.2856,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MGSM",
      "metric_label": "8-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "28.56",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "minerva-math",
    "score": 0.6206,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "Minerva Math",
      "metric_label": "0-shot, CoT",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "62.06",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mmlu",
    "score": 0.6598,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MMLU' (5-shot), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU",
      "metric_label": "5-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "65.98",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mmlu-pro",
    "score": 0.445,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU-Pro",
      "metric_label": "5-shot, CoT",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "44.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "mmmlu",
    "score": 0.5514,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MMMLU",
      "metric_label": "5-shot",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "55.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-0-micro",
    "bench": "multipl-e",
    "score": 0.4921,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "source_id": "card2-ibm-granite-granite-4-0-h-small",
    "by": "lab",
    "date": "2025-11-03",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card, Evaluation Results table (identical table in the granite-4.0-micro card; 'Micro Dense' = Granite 4.0 Micro, 'H Small MoE' = Granite 4.0 H Small; H Micro/H Tiny columns not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column 'Micro Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MULTIPLE",
      "metric_label": "pass@1",
      "column_label": "Micro Dense",
      "column_index": 2,
      "raw_value": "49.21",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "bbh",
    "score": 0.8051,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'BBH' (3-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "BBH",
      "metric_label": "3-shot, CoT",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "80.51",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "bfcl-v3",
    "score": 0.6827,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'BFCL v3', section 'Tool Calling Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Tool Calling Tasks",
      "heading": "print output",
      "row_label": "BFCL v3",
      "metric_label": "",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "68.27",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "bigcodebench",
    "score": 0.35,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "BigCodeBench",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "35.00",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "cruxeval-o",
    "score": 0.4763,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "CRUXEval-O",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "47.63",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "deepmind-math",
    "score": 0.8007,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "DeepMind Math",
      "metric_label": "0-shot, CoT",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "80.07",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "gpqa-unspecified",
    "score": 0.4196,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "GPQA",
      "metric_label": "0-shot, CoT",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "41.96",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "gsm-symbolic",
    "score": 0.837,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'GSM Symbolic' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM Symbolic",
      "metric_label": "8-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "83.70",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "gsm8k",
    "score": 0.9249,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'GSM8K' (8-shot), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "GSM8K",
      "metric_label": "8-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "92.49",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "humaneval",
    "score": 0.8537,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'HumanEval' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "85.37",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "humaneval-plus",
    "score": 0.7988,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'HumanEval+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "HumanEval+",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "79.88",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "ifeval-unspecified",
    "score": 0.8706,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'IFEval Avg', section 'Alignment Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Alignment Tasks",
      "heading": "print output",
      "row_label": "IFEval Avg",
      "metric_label": "",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "87.06",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "include",
    "score": 0.5889,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "INCLUDE",
      "metric_label": "5-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "58.89",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mbpp",
    "score": 0.873,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MBPP' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "87.30",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mbpp-plus",
    "score": 0.7381,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MBPP+' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MBPP+",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "73.81",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mgsm-5lang",
    "score": 0.8232,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MGSM",
      "metric_label": "8-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "82.32",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "minerva-math",
    "score": 0.801,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Math Tasks",
      "heading": "print output",
      "row_label": "Minerva Math",
      "metric_label": "0-shot, CoT",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "80.10",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mmlu",
    "score": 0.7384,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MMLU' (5-shot), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU",
      "metric_label": "5-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "73.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mmlu-pro",
    "score": 0.5599,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "MMLU-Pro",
      "metric_label": "5-shot, CoT",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "55.99",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "mmmlu",
    "score": 0.6484,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual Tasks",
      "heading": "print output",
      "row_label": "MMMLU",
      "metric_label": "5-shot",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "64.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "multipl-e",
    "score": 0.6026,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'MULTIPLE' (pass@1), section 'Code Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Code Tasks",
      "heading": "print output",
      "row_label": "MULTIPLE",
      "metric_label": "pass@1",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "60.26",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-1-8b",
    "bench": "simpleqa",
    "score": 0.0482,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "source_id": "card2-ibm-granite-granite-4-1-8b",
    "by": "lab",
    "date": "2026-05-04",
    "mode": "instruct",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '8B Dense' column (3B/30B Dense are other Granite 4.1 sizes not in the catalogue). Card language table: MMMLU 5-shot over 11 languages (ar, de, en, es, fr, ja, ko, pt, zh, bn, hi); INCLUDE 5-shot over 14 languages; MGSM 8-shot over en, es, fr, ja, zh. Shots/CoT per the card's Metric column. Aggregates (Eval+ Avg, IFEval average of the two strict rows), AlpacaEval 2.0 (LC not stated), ArenaHard (version not stated), MT-Bench and safety rows not used. Column '8B Dense', row 'SimpleQA', section 'General Tasks'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "General Tasks",
      "heading": "print output",
      "row_label": "SimpleQA",
      "metric_label": "",
      "column_label": "8B Dense",
      "column_index": 3,
      "raw_value": "4.82",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "aime-2025",
    "score": 0.8917,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'AIME25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "AIME25",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "89.17",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "arena-hard-v2",
    "score": 0.6793,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'Arena-Hard-V2', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "Arena-Hard-V2",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "67.93",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "bfcl-v4",
    "score": 0.6139,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'BFCL (v4)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "BFCL (v4)",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "61.39",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "birdbench",
    "score": 0.4185,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'BirdBench', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "BirdBench",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "41.85",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "gpqa-unspecified",
    "score": 0.6641,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'GPQA', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "GPQA",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "66.41",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "hmmt-feb-2025",
    "score": 0.8917,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'HMMT Feb25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "HMMT Feb25",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "89.17",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "ifbench-prompt",
    "score": 0.7717,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'IFBench (prompt)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "IFBench (prompt)",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "77.17",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.7577,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'LiveCodeBench v6', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "LiveCodeBench v6",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "75.77",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "mmlu-pro",
    "score": 0.776,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'MMLU-Pro', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-Pro",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "77.60",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "mmlu-prox-lite",
    "score": 0.6664,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'MMLU-ProX lite (IBM)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-ProX lite (IBM)",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "66.64",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "profbench",
    "score": 0.429,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'ProfBench', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "ProfBench",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "42.90",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "ruler-128k",
    "score": 0.8138,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'RULER 128K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 128K",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "81.38",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "ruler-64k",
    "score": 0.8996,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'RULER 64K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 64K",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "89.96",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "scicode-unspecified",
    "score": 0.3876,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'SciCode', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "SciCode",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "38.76",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "swe-bench-multilingual",
    "score": 0.4189,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'SWE Bench Multilingual', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Multilingual",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "41.89",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.3329,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'SWE Bench Pro', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Pro",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "33.29",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "swe-bench-verified",
    "score": 0.57,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'SWE Bench Verified', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Verified",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "57",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "tau3-bench",
    "score": 0.62,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'τ³-bench (AVG)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "τ³-bench (AVG)",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "62.00",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-30b",
    "bench": "terminal-bench-2.1",
    "score": 0.2924,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '30B Dense', row 'Terminal-Bench 2.1', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "Terminal-Bench 2.1",
      "column_label": "30B Dense",
      "column_index": 3,
      "raw_value": "29.24",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "aime-2025",
    "score": 0.7833,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'AIME25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "AIME25",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "78.33",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "arena-hard-v2",
    "score": 0.3496,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'Arena-Hard-V2', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "Arena-Hard-V2",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "34.96",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "bfcl-v4",
    "score": 0.5241,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'BFCL (v4)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "BFCL (v4)",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "52.41",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "gpqa-unspecified",
    "score": 0.548,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'GPQA', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "GPQA",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "54.80",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "hmmt-feb-2025",
    "score": 0.6667,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'HMMT Feb25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "HMMT Feb25",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "66.67",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ifbench-prompt",
    "score": 0.7433,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'IFBench (prompt)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "IFBench (prompt)",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "74.33",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.6971,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'LiveCodeBench v6', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "LiveCodeBench v6",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "69.71",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "mmlu-pro",
    "score": 0.6784,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'MMLU-Pro', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-Pro",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "67.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "mmlu-prox-lite",
    "score": 0.2778,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'MMLU-ProX lite (IBM)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-ProX lite (IBM)",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "27.78",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "profbench",
    "score": 0.321,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'ProfBench', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "ProfBench",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "32.10",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ruler-128k",
    "score": 0.553,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'RULER 128K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 128K",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "55.30",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ruler-64k",
    "score": 0.6752,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'RULER 64K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 64K",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "67.52",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "scicode-unspecified",
    "score": 0.2411,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'SciCode', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "SciCode",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "24.11",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-3b",
    "bench": "tau3-bench",
    "score": 0.4578,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '3B Dense', row 'τ³-bench (AVG)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "τ³-bench (AVG)",
      "column_label": "3B Dense",
      "column_index": 1,
      "raw_value": "45.78",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "aime-2025",
    "score": 0.8667,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'AIME25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "AIME25",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "86.67",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "arena-hard-v2",
    "score": 0.6519,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'Arena-Hard-V2', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "Arena-Hard-V2",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "65.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "bfcl-v4",
    "score": 0.5239,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'BFCL (v4)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "BFCL (v4)",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "52.39",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "birdbench",
    "score": 0.4107,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'BirdBench', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "BirdBench",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "41.07",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "gpqa-unspecified",
    "score": 0.6414,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'GPQA', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "GPQA",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "64.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "hmmt-feb-2025",
    "score": 0.7833,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'HMMT Feb25', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "HMMT Feb25",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "78.33",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "ifbench-prompt",
    "score": 0.7933,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'IFBench (prompt)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "IFBench (prompt)",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "79.33",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.7324,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'LiveCodeBench v6', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "LiveCodeBench v6",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "73.24",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "mmlu-pro",
    "score": 0.7404,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'MMLU-Pro', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-Pro",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "74.04",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "mmlu-prox-lite",
    "score": 0.6106,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'MMLU-ProX lite (IBM)', section 'Chat & Instruction Following'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Chat & Instruction Following",
      "heading": "Evaluation Results",
      "row_label": "MMLU-ProX lite (IBM)",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "61.06",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "profbench",
    "score": 0.412,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'ProfBench', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "ProfBench",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "41.20",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "ruler-128k",
    "score": 0.7141,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'RULER 128K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 128K",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "71.41",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "ruler-64k",
    "score": 0.8099,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'RULER 64K', section 'Long Context'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Long Context",
      "heading": "Evaluation Results",
      "row_label": "RULER 64K",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "80.99",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "scicode-unspecified",
    "score": 0.3609,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'SciCode', section 'Reasoning'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Reasoning",
      "heading": "Evaluation Results",
      "row_label": "SciCode",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "36.09",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "swe-bench-multilingual",
    "score": 0.3078,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'SWE Bench Multilingual', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Multilingual",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "30.78",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.1911,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'SWE Bench Pro', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Pro",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "19.11",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "swe-bench-verified",
    "score": 0.4767,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'SWE Bench Verified', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "SWE Bench Verified",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "47.67",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "tau3-bench",
    "score": 0.5806,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'τ³-bench (AVG)', section 'Agentic (General)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (General)",
      "heading": "Evaluation Results",
      "row_label": "τ³-bench (AVG)",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "58.06",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-4-2-8b",
    "bench": "terminal-bench-2.1",
    "score": 0.2056,
    "source": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md",
    "source_id": "card2-ibm-granite-granite-4-2-30b",
    "by": "lab",
    "date": "2026-09-04",
    "mode": "thinking (default)",
    "note": "IBM own evaluation in the pinned granite-4.2-30b model card, Evaluation Results table (identical table in the 3B and 8B cards; 3B/8B/30B Dense columns are the Granite 4.2 3B/8B/30B releases). Evaluations run with a framework based on NVIDIA NeMo Evaluator SDK; models use thinking mode by default; recommended temperature=1.0, top_p=0.95. The plain 'τ³-bench' row is commented out in the README source and not used; GDPval rating not used; 'NA' cells skipped. Column '8B Dense', row 'Terminal-Bench 2.1', section 'Agentic (Coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Agentic (Coding)",
      "heading": "Evaluation Results",
      "row_label": "Terminal-Bench 2.1",
      "column_label": "8B Dense",
      "column_index": 2,
      "raw_value": "20.56",
      "scale_divisor": 100
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "aime-2024",
    "score": 0.8614,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "source_id": "card2-mistralai-magistral-small-2509",
    "by": "lab",
    "date": "2026-02-23",
    "mode": "reasoning",
    "note": "Mistral own evaluation in the pinned Magistral-Small-2509 model card, Benchmark Results table ('Magistral Small 1.2' = Magistral-Small-2509). Other Magistral rows are different checkpoints. Column 'Magistral Small 1.2', row 'AIME24 pass@1'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark Results",
      "heading": "Benchmark Results",
      "row_label": "AIME24 pass@1",
      "column_label": "Magistral Small 1.2",
      "column_index": 4,
      "raw_value": "86.14%",
      "scale_divisor": 100
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "aime-2025",
    "score": 0.7734,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "source_id": "card2-mistralai-magistral-small-2509",
    "by": "lab",
    "date": "2026-02-23",
    "mode": "reasoning",
    "note": "Mistral own evaluation in the pinned Magistral-Small-2509 model card, Benchmark Results table ('Magistral Small 1.2' = Magistral-Small-2509). Other Magistral rows are different checkpoints. Column 'Magistral Small 1.2', row 'AIME25 pass@1'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark Results",
      "heading": "Benchmark Results",
      "row_label": "AIME25 pass@1",
      "column_label": "Magistral Small 1.2",
      "column_index": 4,
      "raw_value": "77.34%",
      "scale_divisor": 100
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "gpqa-diamond",
    "score": 0.7007,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "source_id": "card2-mistralai-magistral-small-2509",
    "by": "lab",
    "date": "2026-02-23",
    "mode": "reasoning",
    "note": "Mistral own evaluation in the pinned Magistral-Small-2509 model card, Benchmark Results table ('Magistral Small 1.2' = Magistral-Small-2509). Other Magistral rows are different checkpoints. Column 'Magistral Small 1.2', row 'GPQA Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark Results",
      "heading": "Benchmark Results",
      "row_label": "GPQA Diamond",
      "column_label": "Magistral Small 1.2",
      "column_index": 4,
      "raw_value": "70.07%",
      "scale_divisor": 100
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.7088,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "source_id": "card2-mistralai-magistral-small-2509",
    "by": "lab",
    "date": "2026-02-23",
    "mode": "reasoning",
    "note": "Mistral own evaluation in the pinned Magistral-Small-2509 model card, Benchmark Results table ('Magistral Small 1.2' = Magistral-Small-2509). Other Magistral rows are different checkpoints. Column 'Magistral Small 1.2', row 'Livecodebench (v5)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark Results",
      "heading": "Benchmark Results",
      "row_label": "Livecodebench (v5)",
      "column_label": "Magistral Small 1.2",
      "column_index": 4,
      "raw_value": "70.88%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "arena-hard-v2",
    "score": 0.431,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card (Instruction Following / Chat / Tone table). Small 3.1 column is a different checkpoint and not used. Column 'Small 3.2 24B Instruct', row 'Arena Hard v2'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following / Chat / Tone",
      "heading": "Instruction Following / Chat / Tone",
      "row_label": "Arena Hard v2",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "43.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "chartqa",
    "score": 0.874,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, Vision table (MathVista split and AI2D split not stated; not used). Column 'Small 3.2 24B Instruct', row 'ChartQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Vision",
      "heading": "Vision",
      "row_label": "ChartQA",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "87.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "docvqa",
    "score": 0.9486,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, Vision table (MathVista split and AI2D split not stated; not used). Column 'Small 3.2 24B Instruct', row 'DocVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Vision",
      "heading": "Vision",
      "row_label": "DocVQA",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "94.86%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "gpqa-diamond",
    "score": 0.4613,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'GPQA Diamond (5-shot CoT )'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "GPQA Diamond (5-shot CoT )",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "46.13%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "gpqa-main",
    "score": 0.4422,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'GPQA Main (5-shot CoT)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "GPQA Main (5-shot CoT)",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "44.22%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "humaneval-plus-pass5",
    "score": 0.929,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'HumanEval Plus - Pass@5'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "HumanEval Plus - Pass@5",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "92.90%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "math-subset-unspecified",
    "score": 0.6942,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'MATH'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "MATH",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "69.42%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "mbpp-plus-pass5",
    "score": 0.7833,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'MBPP Plus - Pass@5'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "MBPP Plus - Pass@5",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "78.33%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "mmlu",
    "score": 0.805,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'MMLU'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "MMLU",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "80.50%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "mmlu-pro",
    "score": 0.6906,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'MMLU Pro (5-shot CoT)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "MMLU Pro (5-shot CoT)",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "69.06%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "mmmu",
    "score": 0.625,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, Vision table (MathVista split and AI2D split not stated; not used). Column 'Small 3.2 24B Instruct', row 'MMMU'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "Vision",
      "heading": "Vision",
      "row_label": "MMMU",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "62.50%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "simpleqa",
    "score": 0.121,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card, STEM table. MMLU-Pro and GPQA 5-shot CoT; MBPP+/HumanEval+ pass@5; MATH subset not stated. Column 'Small 3.2 24B Instruct', row 'SimpleQA (TotalAcc)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "STEM",
      "heading": "STEM",
      "row_label": "SimpleQA (TotalAcc)",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "12.10%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "wildbench-v2",
    "score": 0.6533,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506",
    "by": "lab",
    "date": "2025-12-22",
    "mode": "instruct",
    "note": "Mistral own evaluation in the pinned Mistral-Small-3.2-24B-Instruct-2506 model card (Instruction Following / Chat / Tone table). Small 3.1 column is a different checkpoint and not used. Column 'Small 3.2 24B Instruct', row 'Wildbench v2'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Instruction Following / Chat / Tone",
      "heading": "Instruction Following / Chat / Tone",
      "row_label": "Wildbench v2",
      "column_label": "Small 3.2 24B Instruct",
      "column_index": 2,
      "raw_value": "65.33%",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "ami-wer",
    "score": 0.1601,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "AMI",
      "column_index": 3,
      "raw_value": "16.01",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "covost2-13lang-avg-wer",
    "score": 0.0885,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Multilingual ASR table; WERs exclude punctuation and capitalisation errors; card states language counts only.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Automatic Speech Recognition (ASR)",
      "row_label": "`Canary-1b-v2`",
      "column_label": "CoVoST-13 Langs",
      "column_index": 2,
      "raw_value": "8.85%",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "earnings22-longform-wer",
    "score": 0.1378,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Long-form inference table: dynamic chunking with 1-second overlap; WER excludes punctuation and capitalisation errors.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "Long-form Inference",
      "row_label": "Earnings-22",
      "column_label": "WER ↓",
      "column_index": 1,
      "raw_value": "13.78%",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "earnings22-wer",
    "score": 0.1179,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "Earnings22",
      "column_index": 7,
      "raw_value": "11.79",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.084,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Multilingual ASR table; WERs exclude punctuation and capitalisation errors; card states language counts only.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Automatic Speech Recognition (ASR)",
      "row_label": "`Canary-1b-v2`",
      "column_label": "Fleurs-25 Langs",
      "column_index": 1,
      "raw_value": "8.40%",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "gigaspeech-wer",
    "score": 0.1082,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "GigaSpeech",
      "column_index": 4,
      "raw_value": "10.82",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0218,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "LS Clean",
      "column_index": 5,
      "raw_value": "2.18",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "librispeech-test-other-wer",
    "score": 0.0356,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "LS Other",
      "column_index": 6,
      "raw_value": "3.56",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "mls-6lang-avg-wer",
    "score": 0.0727,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Multilingual ASR table; WERs exclude punctuation and capitalisation errors; card states language counts only.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Automatic Speech Recognition (ASR)",
      "row_label": "`Canary-1b-v2`",
      "column_label": "MLS - 6 Langs",
      "column_index": 3,
      "raw_value": "7.27%",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "spgispeech-wer",
    "score": 0.0228,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "SPGISpech",
      "column_index": 8,
      "raw_value": "2.28",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "tedlium-wer",
    "score": 0.0429,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "Tedlium",
      "column_index": 9,
      "raw_value": "4.29",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "voxpopuli-en-wer",
    "score": 0.0625,
    "source": "https://huggingface.co/nvidia/canary-1b-v2/blob/d455706339a6b32e1aa40f82c713a482a0c938e2/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-31",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of Canary-1b-v2 (Benchmark Results section). Converted from percent. Hugging Face Open ASR Leaderboard table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Hugging Face Open ASR Leaderboard",
      "row_label": "`Canary-1b-v2`",
      "column_label": "Voxpopuli",
      "column_index": 10,
      "raw_value": "6.25",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "ami-wer",
    "score": 0.1018,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "AMI",
      "column_index": 4,
      "raw_value": "10.18",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "earnings22-wer",
    "score": 0.1042,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "Earnings22",
      "column_index": 8,
      "raw_value": "10.42",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "gigaspeech-wer",
    "score": 0.0941,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "GigaSpeech",
      "column_index": 5,
      "raw_value": "9.41",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.016,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "LS Clean",
      "column_index": 6,
      "raw_value": "1.60",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "librispeech-test-other-wer",
    "score": 0.031,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "LS Other",
      "column_index": 7,
      "raw_value": "3.10",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "spgispeech-wer",
    "score": 0.019,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "SPGISpech",
      "column_index": 9,
      "raw_value": "1.90",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "tedlium-wer",
    "score": 0.0272,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "Tedlium",
      "column_index": 10,
      "raw_value": "2.72",
      "scale_divisor": 100
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0566,
    "source": "https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'ASR Performance (w/o PnC)' table, row Canary-Qwen-2.5B version 2.5.0: greedy decoding; ground truth and prediction normalised with whisper-normalizer 0.1.12; HF Open ASR Leaderboard datasets. Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "ASR Performance (w/o PnC)",
      "row_label": "2.5.0",
      "column_label": "Voxpopuli",
      "column_index": 11,
      "raw_value": "5.66",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "ami-wer",
    "score": 0.171,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "AMI (Meetings test)",
      "anchor": "name: AMI (Meetings test)",
      "raw_value": "17.10",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "commonvoice-9-en-wer",
    "score": 0.0579,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "Mozilla Common Voice 9.0 (en, test)",
      "anchor": "name: Mozilla Common Voice 9.0",
      "raw_value": "5.79",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "earnings22-wer",
    "score": 0.1411,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "Earnings-22",
      "anchor": "name: Earnings-22",
      "raw_value": "14.11",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "gigaspeech-wer",
    "score": 0.0996,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "GigaSpeech",
      "anchor": "name: GigaSpeech",
      "raw_value": "9.96",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0146,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "LibriSpeech (clean)",
      "anchor": "name: LibriSpeech (clean)",
      "raw_value": "1.46",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0247,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "LibriSpeech (other)",
      "anchor": "name: LibriSpeech (other)",
      "raw_value": "2.47",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "spgispeech-wer",
    "score": 0.0311,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "SPGI Speech",
      "anchor": "name: SPGI Speech",
      "raw_value": "3.11",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "tedlium-wer",
    "score": 0.0392,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "tedlium-v3 (release1 config, test)",
      "anchor": "name: tedlium-v3",
      "raw_value": "3.92",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0539,
    "source": "https://huggingface.co/nvidia/parakeet-rnnt-1.1b/blob/2acc4c61eede2f52ddefe74935de1930a9064d4a/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "greedy",
    "note": "NVIDIA NeMo/Suno.ai own evaluation of parakeet-rnnt-1.1b, greedy decoding without external LM (card Performance section). Value taken from the README model-index metadata because the body table header omits the LS test-other column (its nine values align with the model-index order). Converted from percent.",
    "evidence": {
      "kind": "model-index-metadata",
      "dataset_name": "Vox Populi (en, test)",
      "anchor": "name: Vox Populi",
      "raw_value": "5.39",
      "scale_divisor": 100,
      "section": "README front-matter model-index"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "ami-wer",
    "score": 0.1116,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "AMI",
      "column_index": 2,
      "raw_value": "11.16",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "earnings22-wer",
    "score": 0.1115,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "Earnings-22",
      "column_index": 3,
      "raw_value": "11.15",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "gigaspeech-wer",
    "score": 0.0974,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "GigaSpeech",
      "column_index": 4,
      "raw_value": "9.74",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0169,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "LS test-clean",
      "column_index": 5,
      "raw_value": "1.69",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "librispeech-test-other-wer",
    "score": 0.0319,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "LS test-other",
      "column_index": 6,
      "raw_value": "3.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "spgispeech-wer",
    "score": 0.0217,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "SPGI Speech",
      "column_index": 7,
      "raw_value": "2.17",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "tedlium-wer",
    "score": 0.0338,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "TEDLIUM-v3",
      "column_index": 8,
      "raw_value": "3.38",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "voxpopuli-en-wer",
    "score": 0.0595,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-06-29",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation, 'Base Performance' table, row parakeet-tdt-0.6b-v2: Transducer decoder, greedy decoding, no external LM, HF Open ASR Leaderboard datasets (clean audio). Converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Base Performance",
      "row_label": "parakeet-tdt-0.6b-v2",
      "column_label": "VoxPopuli",
      "column_index": 9,
      "raw_value": "5.95",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "ami-wer",
    "score": 0.1131,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "AMI",
      "column_index": 2,
      "raw_value": "11.31%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "covost2-13lang-avg-wer",
    "score": 0.1198,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. Multilingual ASR table 'Average WER' row; WER after removing punctuation and capitalisation from reference and prediction; per-language values in the same table define the language sets.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual ASR",
      "row_label": "Average WER ↓",
      "column_label": "CoVoST",
      "column_index": 3,
      "raw_value": "*11.98%*",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "earnings22-wer",
    "score": 0.1142,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "Earnings-22",
      "column_index": 3,
      "raw_value": "11.42%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.1197,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. Multilingual ASR table 'Average WER' row; WER after removing punctuation and capitalisation from reference and prediction; per-language values in the same table define the language sets.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual ASR",
      "row_label": "Average WER ↓",
      "column_label": "Fleurs",
      "column_index": 1,
      "raw_value": "*11.97%*",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0485,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. Multilingual ASR table, FLEURS English row; punctuation and capitalisation removed.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual ASR",
      "row_label": "en",
      "column_label": "Fleurs",
      "column_index": 1,
      "raw_value": "4.85%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "gigaspeech-wer",
    "score": 0.0959,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "GigaSpeech",
      "column_index": 4,
      "raw_value": "9.59%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0193,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "LS test-clean",
      "column_index": 5,
      "raw_value": "1.93%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0359,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "LS test-other",
      "column_index": 6,
      "raw_value": "3.59%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "mls-6lang-avg-wer",
    "score": 0.0783,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. Multilingual ASR table 'Average WER' row; WER after removing punctuation and capitalisation from reference and prediction; per-language values in the same table define the language sets.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Multilingual ASR",
      "row_label": "Average WER ↓",
      "column_label": "MLS",
      "column_index": 2,
      "raw_value": "*7.83%*",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "spgispeech-wer",
    "score": 0.0397,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "SPGI Speech",
      "column_index": 7,
      "raw_value": "3.97%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "tedlium-wer",
    "score": 0.0275,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "TEDLIUM-v3",
      "column_index": 8,
      "raw_value": "2.75%",
      "scale_divisor": 100
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "voxpopuli-en-wer",
    "score": 0.0614,
    "source": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3/blob/541d1f99c6b0c3cd0b11a95167540bb8edefd82b/README.md",
    "source_id": "first-party-nvidia-asr-cards",
    "by": "lab",
    "date": "2026-08-05",
    "mode": "greedy",
    "note": "NVIDIA's own evaluation of parakeet-tdt-0.6b-v3: Transducer decoder, greedy decoding, no external LM. Converted from percent. HF Open ASR Leaderboard datasets table.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "Huggingface Open-ASR-Leaderboard",
      "row_label": "`parakeet-tdt-0.6b-v3`",
      "column_label": "VoxPopuli",
      "column_index": 9,
      "raw_value": "6.14%",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-m-1",
    "bench": "swe-bench-pro",
    "score": 0.492,
    "source": "https://huggingface.co/poolside/Laguna-M.1/blob/2bf8a4ab6d2e8e1e0d92a41318532a0fee1640d6/README.md#benchmark-results",
    "source_id": "card2-poolside-laguna-m-1",
    "by": "lab",
    "date": "2026-07-14",
    "mode": "thinking",
    "note": "Poolside own evaluation in the pinned Laguna-M.1 model card, Benchmark results table, Laguna M.1 row (comparison-model rows are other labs' published scores and are not used). All benchmarking with Poolside's pool agent harness, max 500 steps, sandboxed (8 GB RAM/2 CPUs; Terminal-Bench 2.0 48 GB/32 CPUs), temperature=1.0, top_k=20, thinking enabled, 256K context; some task images/verifiers patched for infrastructure reliability; mean pass@1 over 4 runs for each benchmark. Row 'Laguna M.1', column 'SWE-bench Pro (Public Dataset)'.",
    "evidence": {
      "kind": "table-transposed",
      "table_index": 0,
      "heading": "Benchmark results",
      "row_label": "Laguna M.1",
      "column_label": "SWE-bench Pro (Public Dataset)",
      "column_index": 4,
      "raw_value": "49.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-pro",
    "score": 0.463,
    "source": "https://huggingface.co/poolside/Laguna-XS.2/blob/a397bde04501591d0ac3d74a5d9d0a4ec298ffb8/README.md#benchmark-results",
    "source_id": "card2-poolside-laguna-xs-2",
    "by": "lab",
    "date": "2026-09-22",
    "mode": "thinking",
    "note": "Poolside own evaluation in the pinned Laguna-XS.2 model card, Benchmark results table, Laguna XS.2 row (comparison rows not used). Laude Institute Harbor framework with Poolside's agent harness, max 500 steps, sandboxed 8 GB RAM/2 CPUs (Terminal-Bench 2.0 48 GB/32 CPUs), temperature=0.7, top_k=20; some task images patched; mean pass@1 over 4 runs (SWE-bench Verified), 7 (Multilingual), 3 (Pro) and 5 (Terminal-Bench 2.0). Row 'Laguna XS.2', column 'SWE-bench Pro (Public Dataset)'.",
    "evidence": {
      "kind": "table-transposed",
      "table_index": 0,
      "heading": "Benchmark results",
      "row_label": "Laguna XS.2",
      "column_label": "SWE-bench Pro (Public Dataset)",
      "column_index": 4,
      "raw_value": "46.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen-image",
    "bench": "dpg-bench",
    "score": 0.8832,
    "source": "https://huggingface.co/Qwen/Qwen-Image/blob/75e0b4be04f60ec59a75f475837eced720f823b6/README.md",
    "source_id": "card2-qwen-qwen-image",
    "by": "lab",
    "date": "2025-08-18",
    "mode": "t2i",
    "note": "Qwen team's own result: purple 'Qwen-Image' bar label in the benchmark overview chart embedded in the pinned Apache-2.0 README (bench.png, publisher-hosted asset archived and hashed). Generation settings, prompt enhancement and sampling are not stated in the card. Competitor bars not used. Editing-panel values (ImgEdit/GEdit/GSO) not recorded because the released editing checkpoint is Qwen-Image-Edit, a separate catalogue model; GenEval not recorded because the card does not state whether the value uses prompt rewriting or an RL variant. Text-rendering sub-scores (OneIG-Text, ChineseWord, TextCraft) not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/bench.png",
      "asset_pinned": false,
      "asset_file": "Qwen__Qwen-Image@75e0b4be04f60ec59a75f475837eced720f823b6.qwen-image-bench.png",
      "asset_sha256": "11b1082da51b27f2a6a3c5d1d77aac7871d86eef7461e5f3d795d63e1e56de6e",
      "section": "(a) Image generation and editing benchmarks / Generation",
      "row_label": "DPG",
      "column_label": "Qwen-Image",
      "raw_value": "88.32",
      "scale_divisor": 100,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "qwen-image",
    "bench": "longtext-cn",
    "score": 0.946,
    "source": "https://huggingface.co/Qwen/Qwen-Image/blob/75e0b4be04f60ec59a75f475837eced720f823b6/README.md",
    "source_id": "card2-qwen-qwen-image",
    "by": "lab",
    "date": "2025-08-18",
    "mode": "t2i",
    "note": "Qwen team's own result: purple 'Qwen-Image' bar label in the benchmark overview chart embedded in the pinned Apache-2.0 README (bench.png, publisher-hosted asset archived and hashed). Generation settings, prompt enhancement and sampling are not stated in the card. Competitor bars not used. Editing-panel values (ImgEdit/GEdit/GSO) not recorded because the released editing checkpoint is Qwen-Image-Edit, a separate catalogue model; GenEval not recorded because the card does not state whether the value uses prompt rewriting or an RL variant. Text-rendering sub-scores (OneIG-Text, ChineseWord, TextCraft) not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/bench.png",
      "asset_pinned": false,
      "asset_file": "Qwen__Qwen-Image@75e0b4be04f60ec59a75f475837eced720f823b6.qwen-image-bench.png",
      "asset_sha256": "11b1082da51b27f2a6a3c5d1d77aac7871d86eef7461e5f3d795d63e1e56de6e",
      "section": "(b) Text rendering benchmarks / Chinese Rendering",
      "row_label": "LongText-Bench (ZH)",
      "column_label": "Qwen-Image",
      "raw_value": "0.946",
      "scale_divisor": 1,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "qwen-image",
    "bench": "longtext-en",
    "score": 0.943,
    "source": "https://huggingface.co/Qwen/Qwen-Image/blob/75e0b4be04f60ec59a75f475837eced720f823b6/README.md",
    "source_id": "card2-qwen-qwen-image",
    "by": "lab",
    "date": "2025-08-18",
    "mode": "t2i",
    "note": "Qwen team's own result: purple 'Qwen-Image' bar label in the benchmark overview chart embedded in the pinned Apache-2.0 README (bench.png, publisher-hosted asset archived and hashed). Generation settings, prompt enhancement and sampling are not stated in the card. Competitor bars not used. Editing-panel values (ImgEdit/GEdit/GSO) not recorded because the released editing checkpoint is Qwen-Image-Edit, a separate catalogue model; GenEval not recorded because the card does not state whether the value uses prompt rewriting or an RL variant. Text-rendering sub-scores (OneIG-Text, ChineseWord, TextCraft) not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/bench.png",
      "asset_pinned": false,
      "asset_file": "Qwen__Qwen-Image@75e0b4be04f60ec59a75f475837eced720f823b6.qwen-image-bench.png",
      "asset_sha256": "11b1082da51b27f2a6a3c5d1d77aac7871d86eef7461e5f3d795d63e1e56de6e",
      "section": "(b) Text rendering benchmarks / English Rendering",
      "row_label": "LongText-Bench (EN)",
      "column_label": "Qwen-Image",
      "raw_value": "0.943",
      "scale_divisor": 1,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "qwen-image",
    "bench": "oneig-cn",
    "score": 0.548,
    "source": "https://huggingface.co/Qwen/Qwen-Image/blob/75e0b4be04f60ec59a75f475837eced720f823b6/README.md",
    "source_id": "card2-qwen-qwen-image",
    "by": "lab",
    "date": "2025-08-18",
    "mode": "t2i",
    "note": "Qwen team's own result: purple 'Qwen-Image' bar label in the benchmark overview chart embedded in the pinned Apache-2.0 README (bench.png, publisher-hosted asset archived and hashed). Generation settings, prompt enhancement and sampling are not stated in the card. Competitor bars not used. Editing-panel values (ImgEdit/GEdit/GSO) not recorded because the released editing checkpoint is Qwen-Image-Edit, a separate catalogue model; GenEval not recorded because the card does not state whether the value uses prompt rewriting or an RL variant. Text-rendering sub-scores (OneIG-Text, ChineseWord, TextCraft) not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/bench.png",
      "asset_pinned": false,
      "asset_file": "Qwen__Qwen-Image@75e0b4be04f60ec59a75f475837eced720f823b6.qwen-image-bench.png",
      "asset_sha256": "11b1082da51b27f2a6a3c5d1d77aac7871d86eef7461e5f3d795d63e1e56de6e",
      "section": "(a) Image generation and editing benchmarks / Generation",
      "row_label": "OneIG-Bench-ZH",
      "column_label": "Qwen-Image",
      "raw_value": "0.548",
      "scale_divisor": 1,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "qwen-image",
    "bench": "oneig-en",
    "score": 0.539,
    "source": "https://huggingface.co/Qwen/Qwen-Image/blob/75e0b4be04f60ec59a75f475837eced720f823b6/README.md",
    "source_id": "card2-qwen-qwen-image",
    "by": "lab",
    "date": "2025-08-18",
    "mode": "t2i",
    "note": "Qwen team's own result: purple 'Qwen-Image' bar label in the benchmark overview chart embedded in the pinned Apache-2.0 README (bench.png, publisher-hosted asset archived and hashed). Generation settings, prompt enhancement and sampling are not stated in the card. Competitor bars not used. Editing-panel values (ImgEdit/GEdit/GSO) not recorded because the released editing checkpoint is Qwen-Image-Edit, a separate catalogue model; GenEval not recorded because the card does not state whether the value uses prompt rewriting or an RL variant. Text-rendering sub-scores (OneIG-Text, ChineseWord, TextCraft) not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/bench.png",
      "asset_pinned": false,
      "asset_file": "Qwen__Qwen-Image@75e0b4be04f60ec59a75f475837eced720f823b6.qwen-image-bench.png",
      "asset_sha256": "11b1082da51b27f2a6a3c5d1d77aac7871d86eef7461e5f3d795d63e1e56de6e",
      "section": "(a) Image generation and editing benchmarks / Generation",
      "row_label": "OneIG-Bench-EN",
      "column_label": "Qwen-Image",
      "raw_value": "0.539",
      "scale_divisor": 1,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "aishell2-test-err",
    "score": 0.0271,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "AISHELL-2-test",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "2.71",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.0739,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "CV-en",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "7.39",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "commonvoice-zh-err-version-unspecified",
    "score": 0.0535,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "CV-zh",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "5.35",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "fleurs-en-wer",
    "score": 0.0335,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "Fleurs-en",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "3.35",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "fleurs-zh-err",
    "score": 0.0241,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "Fleurs-zh",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "2.41",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "gigaspeech-wer",
    "score": 0.0845,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "GigaSpeech",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "8.45",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0163,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used. Cell reports 'clean | other'; clean part.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "1.63 | 3.38",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0338,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used. Cell reports 'clean | other'; other part.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "1.63 | 3.38",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "mls-en-wer",
    "score": 0.0458,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "MLS-en",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "4.58",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "tedlium-wer",
    "score": 0.045,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used. TED-LIUM release not stated.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "Tedlium",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "4.50",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0915,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "VoxPopuli",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "9.15",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.0588,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used. Cell reports 'net | meeting'; meeting part.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "4.97 | 5.88",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "wenetspeech-test-net-err",
    "score": 0.0497,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "mode": "offline",
    "note": "Qwen team's own evaluation, column 'Qwen3-ASR-1.7B', 'ASR Benchmarks on Public Datasets (WER ↓)': vLLM, bfloat16, max_new_tokens=1024, greedy decoding, no language parameter specified. Converted from percent. Competitor and Qwen3-ASR-0.6B columns not used. Cell reports 'net | meeting'; net part.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Qwen3-ASR -1.7B",
      "column_index": 8,
      "raw_value": "4.97 | 5.88",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "aider-polyglot",
    "score": 0.618,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '61.8' in row 'Aider-Polyglot' of the own (highlighted) column.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "Aider-Polyglot",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "61.8",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "bfcl-v3",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '68.7' in row 'BFCL-v3' of the own (highlighted) column.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "BFCL-v3",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "68.7",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-live",
    "score": 0.263,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '26.3' in row 'SWE-bench Live' of the own (highlighted) column. Snapshot not stated.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "SWE-bench Live",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "26.3",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-multilingual",
    "score": 0.547,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '54.7' in row 'SWE-bench Multilingual' of the own (highlighted) column.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "54.7",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-verified",
    "score": 0.696,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '69.6' in row 'SWE-bench Verified w/ OpenHands, 500 turns' of the own (highlighted) column. OpenHands scaffold, 500 turns (the 100-turn value 67.0 is not recorded).",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "SWE-bench Verified w/ OpenHands, 500 turns",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "69.6",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "tau-bench-airline",
    "score": 0.6,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '60.0' in row 'TAU-Bench Airline' of the own (highlighted) column.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "TAU-Bench Airline",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "60.0",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "tau-bench-retail",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct/blob/9d90cf8fca1bf7b7acca42d3fc9ae694a2194069/README.md",
    "source_id": "card2-qwen-qwen3-coder-480b-a35b-instruct",
    "by": "lab",
    "date": "2025-08-21",
    "mode": "non-thinking",
    "note": "Qwen own evaluation shown in the publisher chart embedded in the pinned Qwen3-Coder-480B-A35B-Instruct README (https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg; archived copy sha256 4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f in evidence/Qwen3-Coder__qwen3-coder-main.jpg; visually read). Instruct (non-thinking) checkpoint. Competitor columns (Kimi-K2, DeepSeek-V3-0324, Claude Sonnet-4, GPT-4.1) not used; Terminal-Bench (pre-2.0, harness unstated), Multi-SWE-bench mini/flash, Spider2 and Mind2Web not used. Printed label '77.5' in row 'TAU-Bench Retail' of the own (highlighted) column.",
    "evidence": {
      "kind": "chart-label",
      "quote": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "row_label": "TAU-Bench Retail",
      "column_label": "Qwen3-Coder 480B-A35B-Instruct",
      "raw_value": "77.5",
      "scale_divisor": 100,
      "asset_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Coder/qwen3-coder-main.jpg",
      "asset_sha256": "4e5e0f41b66a1a0384f662a8c7749cd0b4e349a4a3bed8b2160d6ded5e81430f",
      "asset_file": "evidence/Qwen3-Coder__qwen3-coder-main.jpg"
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.0124,
    "source": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/fd4b254389122332181a7c3db7f27e918eec64e3/README.md",
    "source_id": "card2-qwen-qwen3-tts-12hz-1-7b-base",
    "by": "lab",
    "date": "2026-01-23",
    "mode": "voice-clone",
    "note": "Qwen team's own evaluation, row Qwen3-TTS-12Hz-1.7B-Base, zero-shot speech generation on the Seed-TTS test set (WER ↓): bfloat16, max_new_tokens=2048, checkpoint default sampling, language='auto'. ASR transcriber not stated in the card. Converted from percent. Competitor and 25Hz/0.6B rows not used.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "Evaluation",
      "row_label": "Qwen3-TTS-12Hz-1.7B-Base",
      "column_label": "test-en",
      "column_index": 2,
      "raw_value": "1.24",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-zh-err-asr-unstated",
    "score": 0.0077,
    "source": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/fd4b254389122332181a7c3db7f27e918eec64e3/README.md",
    "source_id": "card2-qwen-qwen3-tts-12hz-1-7b-base",
    "by": "lab",
    "date": "2026-01-23",
    "mode": "voice-clone",
    "note": "Qwen team's own evaluation, row Qwen3-TTS-12Hz-1.7B-Base, zero-shot speech generation on the Seed-TTS test set (WER ↓): bfloat16, max_new_tokens=2048, checkpoint default sampling, language='auto'. ASR transcriber not stated in the card. Converted from percent. Competitor and 25Hz/0.6B rows not used.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "Evaluation",
      "row_label": "Qwen3-TTS-12Hz-1.7B-Base",
      "column_label": "test-zh",
      "column_index": 1,
      "raw_value": "0.77",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "aime-2025",
    "score": 0.973,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'AIME 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "AIME 2025",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "97.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "browsecomp",
    "score": 0.516,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'BrowseComp'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "BrowseComp",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "51.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "browsecomp-context-managed",
    "score": 0.69,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'BrowseComp (w/ Context Manager)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "BrowseComp (w/ Context Manager)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "69.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "browsecomp-zh",
    "score": 0.669,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'BrowseComp-ZH'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "BrowseComp-ZH",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "66.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "browsecomp-zh-context-managed",
    "score": 0.737,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'BrowseComp-ZH (w/ Context Manager)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "BrowseComp-ZH (w/ Context Manager)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "73.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "gaia-text-only",
    "score": 0.845,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'GAIA (no file)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "GAIA (no file)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "84.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "hmmt-feb-2025",
    "score": 0.984,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'HMMT 2025 (Feb.)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "HMMT 2025 (Feb.)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "98.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "hmmt-nov-2025",
    "score": 0.94,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'HMMT 2025 (Nov.)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "HMMT 2025 (Nov.)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "94.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "imo-answerbench",
    "score": 0.854,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'IMOAnswerBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "IMOAnswerBench",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "85.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.864,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'LiveCodeBench-V6'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "LiveCodeBench-V6",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "86.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "researchrubrics",
    "score": 0.653,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'ResearchRubrics'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "ResearchRubrics",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "65.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "swe-bench-verified",
    "score": 0.744,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'SWE-bench Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "SWE-bench Verified",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "tau2-bench",
    "score": 0.882,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'τ²-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "τ²-Bench",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "88.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "terminal-bench-2",
    "score": 0.51,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'Terminal-Bench 2.0'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "51.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "xbench-deepsearch-2505",
    "score": 0.837,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'xbench-DeepSearch (2025.05)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "xbench-DeepSearch (2025.05)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "83.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "step-3-5-flash",
    "bench": "xbench-deepsearch-2510",
    "score": 0.563,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "source_id": "card2-stepfun-ai-step-3-5-flash",
    "by": "lab",
    "date": "2026-03-17",
    "mode": "thinking",
    "note": "StepFun own evaluation in the pinned Step-3.5-Flash model card, Detailed Benchmarks table. BrowseComp with Context Manager: the agent resets the context and restarts the loop when the effective context exceeds a threshold (not discard-all). Recommended reasoning/agent sampling temperature=1.0, top_p=0.95 (card). Column 'Step 3.5 Flash', row 'xbench-DeepSearch (2025.10)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Detailed Benchmarks",
      "heading": "Detailed Benchmarks",
      "row_label": "xbench-DeepSearch (2025.10)",
      "column_label": "Step 3.5 Flash",
      "column_index": 1,
      "raw_value": "56.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "gedit-bench-overall-lang-unspecified",
    "score": 0.758,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2",
    "by": "lab",
    "date": "2025-12-29",
    "mode": "thinking+reflection",
    "note": "StepFun's own results, row 'Step1x-edit-v1p2 (thinking + reflection)' (the card's usage example enables both thinking and reflection; base and thinking-only rows not recorded). Competitor and v1.1/preview rows not used. GEdit-Bench G_O on the 0–10 scale divided by 10; language split and judge not stated in the card.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "🔥🔥🔥 News!!",
      "row_label": "Step1x-edit-v1p2 (thinking + reflection)",
      "column_label": "GEdit-Bench G_O⬆️",
      "column_index": 3,
      "raw_value": "7.58",
      "scale_divisor": 10
    }
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "kris-bench-overall",
    "score": 0.6093,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2",
    "by": "lab",
    "date": "2025-12-29",
    "mode": "thinking+reflection",
    "note": "StepFun's own results, row 'Step1x-edit-v1p2 (thinking + reflection)' (the card's usage example enables both thinking and reflection; base and thinking-only rows not recorded). Competitor and v1.1/preview rows not used. KRIS-Bench Overall converted from percent.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "🔥🔥🔥 News!!",
      "row_label": "Step1x-edit-v1p2 (thinking + reflection)",
      "column_label": "Kris-Bench Overall⬆️",
      "column_index": 7,
      "raw_value": "60.93",
      "scale_divisor": 100
    }
  },
  {
    "model": "hy3",
    "bench": "apex-agents",
    "score": 0.256,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '25.6' for 'Apex-Agent (pass@1)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "Apex-Agent (pass@1)",
      "column_label": "Hy3",
      "raw_value": "25.6",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "browsecomp-context-managed",
    "score": 0.842,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '84.2' for 'BrowseComp'. Self-summary context management.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "BrowseComp",
      "column_label": "Hy3",
      "raw_value": "84.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "cl-bench",
    "score": 0.238,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '23.8' for 'CL-bench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "CL-bench",
      "column_label": "Hy3",
      "raw_value": "23.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "cl-bench-life",
    "score": 0.17,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '17.0' for 'CL-bench life'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "CL-bench life",
      "column_label": "Hy3",
      "raw_value": "17.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "claw-eval-pass3",
    "score": 0.685,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '68.5' for 'ClawEval (pass^3)'. Internal harness, 20260325 version, 105 queries.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "ClawEval (pass^3)",
      "column_label": "Hy3",
      "raw_value": "68.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "cmt-benchmark",
    "score": 0.378,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '37.8' for 'CMT-Benchmark'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "CMT-Benchmark",
      "column_label": "Hy3",
      "raw_value": "37.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "deepsearchqa",
    "score": 0.91,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '91.0' for 'DeepSearchQA'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "DeepSearchQA",
      "column_label": "Hy3",
      "raw_value": "91.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "frontierscience-olympiad",
    "score": 0.748,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '74.8' for 'FrontierScience-Olympiad'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "FrontierScience-Olympiad",
      "column_label": "Hy3",
      "raw_value": "74.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "frontierscience-research",
    "score": 0.213,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '21.3' for 'FrontierScience-Research'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "FrontierScience-Research",
      "column_label": "Hy3",
      "raw_value": "21.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "gpqa-diamond",
    "score": 0.904,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '90.4' for 'GPQA Diamond'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "GPQA Diamond",
      "column_label": "Hy3",
      "raw_value": "90.4",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "hle",
    "score": 0.37,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '37.0' for 'HLE (no tools, text-only)'. Text-only subset.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "HLE (no tools, text-only)",
      "column_label": "Hy3",
      "raw_value": "37.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "hle-with-tools",
    "score": 0.532,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '53.2' for 'HLE (with tools, text-only)'. Text-only subset.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "HLE (with tools, text-only)",
      "column_label": "Hy3",
      "raw_value": "53.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "imo-answerbench",
    "score": 0.9,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '90.0' for 'IMOAnswerBench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "IMOAnswerBench",
      "column_label": "Hy3",
      "raw_value": "90.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "matharena-apex",
    "score": 0.387,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '38.7' for 'MathArena Apex'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "MathArena Apex",
      "column_label": "Hy3",
      "raw_value": "38.7",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "mcp-atlas",
    "score": 0.791,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '79.1' for 'MCP atlas (public)'. Scale April 2026 methodology (100 tool-call budget), not the older 20-turn limit.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "MCP atlas (public)",
      "column_label": "Hy3",
      "raw_value": "79.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "nl2repo",
    "score": 0.456,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '45.6' for 'NL2repo'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "NL2repo",
      "column_label": "Hy3",
      "raw_value": "45.6",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "phybench",
    "score": 0.774,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '77.4' for 'PHYBench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "PHYBench",
      "column_label": "Hy3",
      "raw_value": "77.4",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "skillsbench-79-text-claude-code",
    "score": 0.553,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '55.3' for 'SkillsBench(79,text-only)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "SkillsBench(79,text-only)",
      "column_label": "Hy3",
      "raw_value": "55.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "superchem",
    "score": 0.549,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '54.9' for 'SuperChem'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "SuperChem",
      "column_label": "Hy3",
      "raw_value": "54.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "swe-bench-multilingual",
    "score": 0.758,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '75.8' for 'SWE-bench Multilingual'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Hy3",
      "raw_value": "75.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.579,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '57.9' for 'SWE-bench Pro'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "SWE-bench Pro",
      "column_label": "Hy3",
      "raw_value": "57.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "swe-bench-verified",
    "score": 0.78,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '78.0' for 'SWE-bench Verified'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "SWE-bench Verified",
      "column_label": "Hy3",
      "raw_value": "78.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "terminal-bench-2.1",
    "score": 0.717,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '71.7' for 'Terminal-Bench 2.1'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "Terminal-Bench 2.1",
      "column_label": "Hy3",
      "raw_value": "71.7",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "toolathlon",
    "score": 0.485,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '48.5' for 'Toolathlon'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "Toolathlon",
      "column_label": "Hy3",
      "raw_value": "48.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "usamo-2026",
    "score": 0.72,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '72.0' for 'USAMO 2026'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "USAMO 2026",
      "column_label": "Hy3",
      "raw_value": "72.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "widesearch",
    "score": 0.764,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '76.4' for 'WideSearch'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "WideSearch",
      "column_label": "Hy3",
      "raw_value": "76.4",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "hy3",
    "bench": "wildclawbench-35-text",
    "score": 0.536,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "source_id": "card2-tencent-hy3",
    "by": "lab",
    "date": "2026-07-20",
    "mode": "highest reasoning effort",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, embedded in the README 'Benchmark Appendix'; visually read; archived copy hashed in evidence). Hy3 column only (Hy3 preview is a different checkpoint; other columns are other labs' models). Appendix notes: reasoning effort set to the highest tier; SWE-bench series with the SWE-agent scaffold; Terminal-Bench 2.1 with Terminus-2 (xml parser, 4h timeout, 16 CPU, 32 GB, 500 episodes); NL2Repo via Claude Code (250 turns, 12,000 s); SkillsBench via Claude Code on 79 text-only tasks averaged over 3 runs; MCP-Atlas with Scale's April 2026 methodology (100 tool calls/task) on the 500-task public set, Gemini 2.5 Pro judge; WildClaw via OpenClaw on 35 text-only queries; Claw Eval internal harness on the 20260325 version (105 queries), Gemini-3.5-flash judge; agentic search via internal harness with self-summary context management for BrowseComp; FrontierScience-Olympiad judged by gpt-oss-120b (high). Internal Hy-* benchmarks, DeepSWE (version unstated), ArxivMath (month unstated), HorizonMath pass@12 and AA-LCR not used. Printed label '53.6' for 'WildClawBench (35, text-only)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark-appendix.png",
      "row_label": "WildClawBench (35, text-only)",
      "column_label": "Hy3",
      "raw_value": "53.6",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/tencent/Hy3/resolve/a960ebc3da325ba167f069f76c41eb62c9280d22/assets/benchmark-appendix.png",
      "asset_sha256": "c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174",
      "asset_file": "evidence/tencent__Hy3__benchmark-appendix.png"
    }
  },
  {
    "model": "inkling-small",
    "bench": "aime-2026",
    "score": 0.955,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'AIME 2026', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "AIME 2026",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "95.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "browsecomp-context-managed",
    "score": 0.774,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'BrowseComp (with context management)', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "BrowseComp (with context management)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "77.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "charxiv-rq",
    "score": 0.774,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'Charxiv RQ (original / with python)', section 'Vision'. Cell '77.4/81.3%': first value used (label order: MCP Atlas public / all; Charxiv RQ original / with python).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Vision",
      "heading": "5. Evaluations",
      "row_label": "Charxiv RQ (original / with python)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "77.4/81.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "critpt",
    "score": 0.083,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'CritPt', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "CritPt",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "8.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "global-mmlu-lite",
    "score": 0.867,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'Global-MMLU-Lite', section 'Chat'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Chat",
      "heading": "5. Evaluations",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "86.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "gpqa-diamond",
    "score": 0.895,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'GPQA Diamond', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "GPQA Diamond",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "89.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "hle",
    "score": 0.316,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'HLE (text only)', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HLE (text only)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "31.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "hle-with-tools",
    "score": 0.478,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'HLE (with tools)', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HLE (with tools)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "47.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "hmmt-feb-2026",
    "score": 0.902,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'HMMT Feb 2026', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HMMT Feb 2026",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "90.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "ifbench-unspecified",
    "score": 0.822,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'IFBench', section 'Chat'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Chat",
      "heading": "5. Evaluations",
      "row_label": "IFBench",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "82.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "mcp-atlas",
    "score": 0.796,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'MCP Atlas (public / all)', section 'Agentic (general)'. Cell '79.6/79.2%': first value used (label order: MCP Atlas public / all; Charxiv RQ original / with python).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "MCP Atlas (public / all)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "79.6/79.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "mmau",
    "score": 0.77,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'MMAU', section 'Audio'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Audio",
      "heading": "5. Evaluations",
      "row_label": "MMAU",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "77.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "mmmu-pro",
    "score": 0.74,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'MMMU Pro (Standard 10)', section 'Vision'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Vision",
      "heading": "5. Evaluations",
      "row_label": "MMMU Pro (Standard 10)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "74.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "scicode-unspecified",
    "score": 0.487,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'SciCode', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SciCode",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "48.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "simpleqa-verified",
    "score": 0.206,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'SimpleQA Verified', section 'Factuality'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Factuality",
      "heading": "5. Evaluations",
      "row_label": "SimpleQA Verified",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "20.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "swe-bench-pro",
    "score": 0.559,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'SWEBench Pro (public)', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SWEBench Pro (public)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "55.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "swe-bench-verified",
    "score": 0.802,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'SWEBench Verified', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SWEBench Verified",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "80.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "tau3-banking",
    "score": 0.155,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'Tau 3 Banking', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "Tau 3 Banking",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "15.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "terminal-bench-2.1",
    "score": 0.647,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'Terminal Bench 2.1 (best harness)', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "Terminal Bench 2.1 (best harness)",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "64.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling-small",
    "bench": "toolathlon-verified",
    "score": 0.544,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling-Small', row 'Toolathlon Verified', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "Toolathlon Verified",
      "column_label": "Inkling-Small",
      "column_index": 1,
      "raw_value": "54.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "aime-2026",
    "score": 0.971,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'AIME 2026', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "AIME 2026",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "97.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "browsecomp-context-managed",
    "score": 0.771,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'BrowseComp (with context management)', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "BrowseComp (with context management)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "77.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "charxiv-rq",
    "score": 0.781,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'Charxiv RQ (original / with python)', section 'Vision'. Cell '78.1/82.0%': first value used (label order: MCP Atlas public / all; Charxiv RQ original / with python).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Vision",
      "heading": "5. Evaluations",
      "row_label": "Charxiv RQ (original / with python)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "78.1/82.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "critpt",
    "score": 0.054,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'CritPt', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "CritPt",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "5.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "global-mmlu-lite",
    "score": 0.887,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'Global-MMLU-Lite', section 'Chat'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Chat",
      "heading": "5. Evaluations",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "88.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "gpqa-diamond",
    "score": 0.872,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'GPQA Diamond', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "GPQA Diamond",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "87.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "hle",
    "score": 0.297,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'HLE (text only)', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HLE (text only)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "29.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "hle-with-tools",
    "score": 0.46,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'HLE (with tools)', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HLE (with tools)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "46.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "hmmt-feb-2026",
    "score": 0.863,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'HMMT Feb 2026', section 'Reasoning (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Reasoning (general)",
      "heading": "5. Evaluations",
      "row_label": "HMMT Feb 2026",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "86.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "ifbench-unspecified",
    "score": 0.798,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'IFBench', section 'Chat'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Chat",
      "heading": "5. Evaluations",
      "row_label": "IFBench",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "79.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "mcp-atlas",
    "score": 0.788,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'MCP Atlas (public / all)', section 'Agentic (general)'. Cell '78.8/76.0%': first value used (label order: MCP Atlas public / all; Charxiv RQ original / with python).",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "MCP Atlas (public / all)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "78.8/76.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "mmau",
    "score": 0.772,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'MMAU', section 'Audio'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Audio",
      "heading": "5. Evaluations",
      "row_label": "MMAU",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "77.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "mmmu-pro",
    "score": 0.735,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'MMMU Pro (Standard 10)', section 'Vision'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Vision",
      "heading": "5. Evaluations",
      "row_label": "MMMU Pro (Standard 10)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "73.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "scicode-unspecified",
    "score": 0.461,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'SciCode', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SciCode",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "46.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "simpleqa-verified",
    "score": 0.439,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'SimpleQA Verified', section 'Factuality'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Factuality",
      "heading": "5. Evaluations",
      "row_label": "SimpleQA Verified",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "43.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "swe-bench-pro",
    "score": 0.543,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'SWEBench Pro (public)', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SWEBench Pro (public)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "54.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "swe-bench-verified",
    "score": 0.776,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'SWEBench Verified', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "SWEBench Verified",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "77.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "tau3-banking",
    "score": 0.237,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'Tau 3 Banking', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "Tau 3 Banking",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "23.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "terminal-bench-2.1",
    "score": 0.638,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'Terminal Bench 2.1 (best harness)', section 'Agentic (coding)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (coding)",
      "heading": "5. Evaluations",
      "row_label": "Terminal Bench 2.1 (best harness)",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "63.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "inkling",
    "bench": "toolathlon-verified",
    "score": 0.455,
    "source": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md",
    "source_id": "card2-thinkingmachines-inkling-small",
    "by": "lab",
    "date": "2026-07-31",
    "mode": "reasoning",
    "note": "Thinking Machines own evaluation in the pinned Inkling-Small model card, Evaluations table (Inkling column is a same-lab evaluation). Footnotes (Inkling-Small card): SWE-bench Verified uses a bash-only harness for Inkling and Inkling-Small; Terminal Bench 2.1 uses Thinking Machines' internal coding harness, with a small number of web-search-contaminated solutions assigned 0; MCP Atlas first value = public set; Charxiv RQ first value = original (without python). AA-labelled rows (AA Index, GDPval-AA, AA-Briefcase, AA Omniscience), ARC-AGI, audio MC/VoiceBench and safety rows not used. Column 'Inkling', row 'Toolathlon Verified', section 'Agentic (general)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Agentic (general)",
      "heading": "5. Evaluations",
      "row_label": "Toolathlon Verified",
      "column_label": "Inkling",
      "column_index": 7,
      "raw_value": "45.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "wan-2-1-14b",
    "bench": "wan-bench-2-1-weighted",
    "score": 0.724,
    "source": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B/blob/a064a6c71f5be440641209c07bf2a5ce7a2ff5e4/README.md",
    "source_id": "card2-wan-ai-wan2-1-t2v-14b",
    "by": "lab",
    "date": "2025-03-12",
    "mode": "t2v",
    "note": "Wan team's own evaluation, 'Comparisons to SOTA' table image (assets/vben_vs_sota.png committed in the pinned repository), column 'Wan-14B' = Wan2.1-T2V-14B, row 'Weighted Score'. 1,035 internal prompts, 14 dimensions / 26 sub-dimensions, human-preference-derived weights. Competitor columns (incl. Mochi) and per-dimension rows not recorded.",
    "evidence": {
      "kind": "chart-label",
      "asset_url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B/blob/a064a6c71f5be440641209c07bf2a5ce7a2ff5e4/assets/vben_vs_sota.png",
      "asset_pinned": true,
      "asset_file": "Wan-AI__Wan2.1-T2V-14B@a064a6c71f5be440641209c07bf2a5ce7a2ff5e4.wan21-vben_vs_sota.png",
      "asset_sha256": "9a0e86ca85046d2675f97984b88b6e74df07bba8a62a31ab8a1aef50d4eda44e",
      "section": "Comparisons to SOTA",
      "row_label": "Weighted Score",
      "column_label": "Wan-14B",
      "raw_value": "0.724",
      "scale_divisor": 1,
      "reviewed": "label read visually from the downloaded image"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "charxiv-rq",
    "score": 0.81,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '81.0' for 'CharXiv RQ'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-multimodal-bench.png",
      "row_label": "CharXiv RQ",
      "column_label": "MiMo-V2.5",
      "raw_value": "81.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-multimodal-bench.png",
      "asset_sha256": "8abf8b5c85b1fe0d131d66e39be6849e4243122001558150b382959946c7b331",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-multimodal-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "dailyomni",
    "score": 0.835,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '83.5' for 'DailyOmni'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-multimodal-bench.png",
      "row_label": "DailyOmni",
      "column_label": "MiMo-V2.5",
      "raw_value": "83.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-multimodal-bench.png",
      "asset_sha256": "8abf8b5c85b1fe0d131d66e39be6849e4243122001558150b382959946c7b331",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-multimodal-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "hr-bench-4k",
    "score": 0.885,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '88.5' for 'HR-Bench (4k)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-multimodal-bench.png",
      "row_label": "HR-Bench (4k)",
      "column_label": "MiMo-V2.5",
      "raw_value": "88.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-multimodal-bench.png",
      "asset_sha256": "8abf8b5c85b1fe0d131d66e39be6849e4243122001558150b382959946c7b331",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-multimodal-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "mmmu-pro",
    "score": 0.779,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '77.9' for 'MMMU-Pro'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-multimodal-bench.png",
      "row_label": "MMMU-Pro",
      "column_label": "MiMo-V2.5",
      "raw_value": "77.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-multimodal-bench.png",
      "asset_sha256": "8abf8b5c85b1fe0d131d66e39be6849e4243122001558150b382959946c7b331",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-multimodal-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.561,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '56.1' for 'SWE-Bench Pro'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-coding-bench.png",
      "row_label": "SWE-Bench Pro",
      "column_label": "MiMo-V2.5",
      "raw_value": "56.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-coding-bench.png",
      "asset_sha256": "a64757621731f35a2dfeb682a29fc59e489eb71339b81a9482852268978e4c3d",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-coding-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "terminal-bench-2",
    "score": 0.658,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '65.8' for 'Terminal-Bench 2.0'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-coding-bench.png",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "MiMo-V2.5",
      "raw_value": "65.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-coding-bench.png",
      "asset_sha256": "a64757621731f35a2dfeb682a29fc59e489eb71339b81a9482852268978e4c3d",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-coding-bench.png"
    }
  },
  {
    "model": "mimo-v2-5",
    "bench": "videoholmes",
    "score": 0.64,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in benchmark charts committed in the pinned XiaomiMiMo/MiMo-V2.5 repository (assets/mimo-v2.5-coding-bench.png and assets/mimo-v2.5-multimodal-bench.png, embedded in README section 3; visually read; archived copies hashed in evidence). MiMo-V2.5 (orange) bars only; MiMo-V2-Pro/V2-Omni are different checkpoints. Settings are not stated in the card. Internal MiMo Coding Bench, Claw-Eval variants, Video-MME (subtitle setting unstated) and OmniDocBench (version/metric unstated) not used. Printed label '64.0' for 'VideoHolmes'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/mimo-v2.5-multimodal-bench.png",
      "row_label": "VideoHolmes",
      "column_label": "MiMo-V2.5",
      "raw_value": "64.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/63651580ca774f8504f676040460aed3e1244ac1/assets/mimo-v2.5-multimodal-bench.png",
      "asset_sha256": "8abf8b5c85b1fe0d131d66e39be6849e4243122001558150b382959946c7b331",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5__mimo-v2.5-multimodal-bench.png"
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "aime-2025",
    "score": 0.916,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'AIME 25'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "AIME 25",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "91.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "browsecomp",
    "score": 0.428,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'BrowseComp'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "BrowseComp",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "42.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "gpqa-unspecified",
    "score": 0.752,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'GPQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "GPQA",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "75.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "hle",
    "score": 0.144,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'HLE'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "HLE",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "14.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.64,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'LCB v6'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "LCB v6",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "64.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "swe-bench-verified",
    "score": 0.592,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'SWE-bench Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "59.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7-flash",
    "bench": "tau2-bench",
    "score": 0.795,
    "source": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md",
    "source_id": "card2-zai-org-glm-4-7-flash",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7-Flash model card. GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Column 'GLM-4.7-Flash', row 'τ²-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Performances on Benchmarks",
      "heading": "Performances on Benchmarks",
      "row_label": "τ²-Bench",
      "column_label": "GLM-4.7-Flash",
      "column_index": 1,
      "raw_value": "79.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "aime-2025",
    "score": 0.939,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'AIME 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "AIME 2025",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "93.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "browsecomp",
    "score": 0.451,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'BrowseComp'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "BrowseComp",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "45.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "browsecomp-context-managed",
    "score": 0.575,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'BrowseComp (w/ Context Manage)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "BrowseComp (w/ Context Manage)",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "57.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "browsecomp-zh",
    "score": 0.495,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'BrowseComp-Zh'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "BrowseComp-Zh",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "49.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "gpqa-diamond",
    "score": 0.81,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'GPQA-Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "GPQA-Diamond",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "hle",
    "score": 0.172,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'HLE'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "HLE",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "17.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "hle-with-tools",
    "score": 0.304,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'HLE (w/ Tools)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "HLE (w/ Tools)",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "30.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "hmmt-feb-2025",
    "score": 0.892,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'HMMT Feb. 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "HMMT Feb. 2025",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "89.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "hmmt-nov-2025",
    "score": 0.877,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'HMMT Nov. 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "HMMT Nov. 2025",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "87.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "imo-answerbench",
    "score": 0.735,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'IMOAnswerBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "IMOAnswerBench",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "73.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.828,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'LiveCodeBench-v6'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "LiveCodeBench-v6",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "mmlu-pro",
    "score": 0.832,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'MMLU-Pro'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "MMLU-Pro",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "83.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "swe-bench-multilingual",
    "score": 0.538,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'SWE-bench Multilingual'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "SWE-bench Multilingual",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "53.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "swe-bench-verified",
    "score": 0.68,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'SWE-bench Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "68.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "tau2-bench",
    "score": 0.752,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'τ²-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "τ²-Bench",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "75.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-6",
    "bench": "terminal-bench-2",
    "score": 0.245,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.6', row 'Terminal Bench 2.0'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "Terminal Bench 2.0",
      "column_label": "GLM-4.6",
      "column_index": 2,
      "raw_value": "24.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "aime-2025",
    "score": 0.957,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.7', row 'AIME 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "AIME 2025",
      "column_label": "GLM-4.7",
      "column_index": 1,
      "raw_value": "95.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "hmmt-feb-2025",
    "score": 0.971,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.7', row 'HMMT Feb. 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "HMMT Feb. 2025",
      "column_label": "GLM-4.7",
      "column_index": 1,
      "raw_value": "97.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.849,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.7', row 'LiveCodeBench-v6'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "LiveCodeBench-v6",
      "column_label": "GLM-4.7",
      "column_index": 1,
      "raw_value": "84.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "mmlu-pro",
    "score": 0.843,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "source_id": "card2-zai-org-glm-4-7",
    "by": "lab",
    "date": "2026-01-29",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card (GLM-4.6 column is a same-lab evaluation). GLM evaluation parameters (card): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes. Terminal Bench Hard (an Artificial Analysis subset) not used. Column 'GLM-4.7', row 'MMLU-Pro'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Introduction",
      "heading": "Introduction",
      "row_label": "MMLU-Pro",
      "column_label": "GLM-4.7",
      "column_index": 1,
      "raw_value": "84.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "aime-2026-i",
    "score": 0.929,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'AIME 2026 I'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "AIME 2026 I",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "92.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp-context-managed",
    "score": 0.675,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'BrowseComp (w/ Context Manage)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp (w/ Context Manage)",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "67.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp-zh",
    "score": 0.666,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'BrowseComp-Zh'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp-Zh",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "66.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "cybergym",
    "score": 0.235,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'CyberGym'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "CyberGym",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "23.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "mcp-atlas",
    "score": 0.52,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'MCP-Atlas (Public Set)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "MCP-Atlas (Public Set)",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "52.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "swe-bench-multilingual",
    "score": 0.667,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'SWE-bench Multilingual'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "SWE-bench Multilingual",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "66.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "swe-bench-verified",
    "score": 0.738,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'SWE-bench Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "73.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "tau2-bench",
    "score": 0.874,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'τ²-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "τ²-Bench",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "terminal-bench-2",
    "score": 0.41,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-4.7', row 'Terminal-Bench 2.0 (Terminus 2)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "Terminal-Bench 2.0 (Terminus 2)",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "41.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "aime-2026-i",
    "score": 0.927,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'AIME 2026 I'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "AIME 2026 I",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "92.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "browsecomp-zh",
    "score": 0.727,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'BrowseComp-Zh'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "BrowseComp-Zh",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "72.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "cybergym",
    "score": 0.432,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'CyberGym'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "CyberGym",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "43.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "mcp-atlas",
    "score": 0.678,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'MCP-Atlas (Public Set)'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "MCP-Atlas (Public Set)",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "swe-bench-multilingual",
    "score": 0.733,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'SWE-bench Multilingual'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "SWE-bench Multilingual",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "73.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "swe-bench-verified",
    "score": 0.778,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'SWE-bench Verified'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "77.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "tau2-bench",
    "score": 0.897,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "source_id": "card2-zai-org-glm-5",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "thinking",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card Benchmark table (GLM-4.7 column is a same-lab evaluation). GLM-5 card footnotes: HLE and reasoning tasks temperature=1.0, top_p=0.95, max_new_tokens=131072, text-only HLE subset by default, GPT-5.2 (medium) judge, HLE-with-tools 202,752 context; SWE-bench suites with OpenHands (temperature=0.7, top_p=0.95, 200K context); BrowseComp keeps the last 5 turns without context management and uses discard-all with it; Terminal-Bench 2.0 (Terminus 2) timeout 2h, temperature=0.7, top_p=1.0, 128K context (first value of 'x / y †' cells is the standard dataset, † = verified dataset not used); CyberGym in Claude Code 2.1.18, 1,507 tasks; MCP-Atlas 500-task public subset, Gemini 3 Pro judge; τ²-bench with retail/telecom prompt adjustment and airline fixes. Column 'GLM-5', row 'τ²-Bench'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "Benchmark",
      "heading": "Benchmark",
      "row_label": "τ²-Bench",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "89.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "bbh",
    "score": 0.857,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'BIG-Bench Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "BIG-Bench Hard",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "85.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "bird-sql-dev",
    "score": 0.479,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'Bird-SQL (dev)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Bird-SQL (dev)",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "47.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "chartqa",
    "score": 0.757,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'ChartQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "ChartQA",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "75.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "docvqa",
    "score": 0.871,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'DocVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "DocVQA",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "87.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "facts-grounding",
    "score": 0.758,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'FACTS Grounding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "FACTS Grounding",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "global-mmlu-lite",
    "score": 0.695,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'Global-MMLU-Lite'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "",
      "heading": "",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "69.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "gpqa-diamond",
    "score": 0.409,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'GPQA Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "GPQA Diamond",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "40.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "gsm8k",
    "score": 0.944,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'GSM8K'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "GSM8K",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "94.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "humaneval",
    "score": 0.854,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'HumanEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "HumanEval",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "85.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "ifeval-unspecified",
    "score": 0.889,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'IFEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "IFEval",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "88.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "infographicvqa",
    "score": 0.649,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'InfoVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "InfoVQA",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "64.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "livecodebench-unversioned",
    "score": 0.246,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'LiveCodeBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "LiveCodeBench",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "24.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "math-subset-unspecified",
    "score": 0.838,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Math",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "83.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "mathvista-mini",
    "score": 0.629,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'MathVista (testmini)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MathVista (testmini)",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "62.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "mbpp",
    "score": 0.73,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'MBPP'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MBPP",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmlu-pro",
    "score": 0.606,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'MMLU (Pro)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MMLU (Pro)",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "60.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmmu",
    "score": 0.596,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'MMMU (val)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MMMU (val)",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "59.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "simpleqa",
    "score": 0.063,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'SimpleQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "SimpleQA",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 4,
      "raw_value": "6.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "textvqa",
    "score": 0.677,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 12B', row 'TextVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "TextVQA",
      "column_label": "Gemma 3 IT 12B",
      "column_index": 2,
      "raw_value": "67.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "bbh",
    "score": 0.391,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'BIG-Bench Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "BIG-Bench Hard",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "39.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "bird-sql-dev",
    "score": 0.064,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'Bird-SQL (dev)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Bird-SQL (dev)",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "6.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "facts-grounding",
    "score": 0.364,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'FACTS Grounding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "FACTS Grounding",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "36.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "global-mmlu-lite",
    "score": 0.342,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'Global-MMLU-Lite'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "",
      "heading": "",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "34.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "gpqa-diamond",
    "score": 0.192,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'GPQA Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "GPQA Diamond",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "19.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "gsm8k",
    "score": 0.628,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'GSM8K'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "GSM8K",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "62.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "humaneval",
    "score": 0.415,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'HumanEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "HumanEval",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "41.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifeval-unspecified",
    "score": 0.802,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'IFEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "IFEval",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "80.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "livecodebench-unversioned",
    "score": 0.019,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'LiveCodeBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "LiveCodeBench",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "1.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "math-subset-unspecified",
    "score": 0.48,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Math",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "48.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "mbpp",
    "score": 0.352,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'MBPP'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MBPP",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "35.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmlu-pro",
    "score": 0.147,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'MMLU (Pro)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MMLU (Pro)",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "14.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "simpleqa",
    "score": 0.022,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 1B', row 'SimpleQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "SimpleQA",
      "column_label": "Gemma 3 IT 1B",
      "column_index": 2,
      "raw_value": "2.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "bbh",
    "score": 0.876,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'BIG-Bench Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "BIG-Bench Hard",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "87.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "bird-sql-dev",
    "score": 0.544,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'Bird-SQL (dev)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Bird-SQL (dev)",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "54.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "chartqa",
    "score": 0.78,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'ChartQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "ChartQA",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 3,
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "docvqa",
    "score": 0.866,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'DocVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "DocVQA",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 3,
      "raw_value": "86.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "facts-grounding",
    "score": 0.749,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'FACTS Grounding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "FACTS Grounding",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "74.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "global-mmlu-lite",
    "score": 0.751,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'Global-MMLU-Lite'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "",
      "heading": "",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "75.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "gsm8k",
    "score": 0.959,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'GSM8K'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "GSM8K",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "95.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "humaneval",
    "score": 0.878,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'HumanEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "HumanEval",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "87.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "livecodebench-unversioned",
    "score": 0.297,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'LiveCodeBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "LiveCodeBench",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "29.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "math-subset-unspecified",
    "score": 0.89,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Math",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "89.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "mathvista-mini",
    "score": 0.676,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'MathVista (testmini)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MathVista (testmini)",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 3,
      "raw_value": "67.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "mbpp",
    "score": 0.744,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'MBPP'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MBPP",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmmu",
    "score": 0.649,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'MMMU (val)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MMMU (val)",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 3,
      "raw_value": "64.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "simpleqa",
    "score": 0.1,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'SimpleQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "SimpleQA",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 5,
      "raw_value": "10.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "textvqa",
    "score": 0.651,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 27B', row 'TextVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "TextVQA",
      "column_label": "Gemma 3 IT 27B",
      "column_index": 3,
      "raw_value": "65.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "bbh",
    "score": 0.722,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'BIG-Bench Hard'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "BIG-Bench Hard",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "72.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "bird-sql-dev",
    "score": 0.363,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'Bird-SQL (dev)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Bird-SQL (dev)",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "36.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "chartqa",
    "score": 0.688,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'ChartQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "ChartQA",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "68.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "docvqa",
    "score": 0.758,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'DocVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "DocVQA",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "facts-grounding",
    "score": 0.701,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'FACTS Grounding'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "FACTS Grounding",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "70.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "global-mmlu-lite",
    "score": 0.545,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'Global-MMLU-Lite'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "",
      "heading": "",
      "row_label": "Global-MMLU-Lite",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "54.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "gpqa-diamond",
    "score": 0.308,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'GPQA Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "GPQA Diamond",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "30.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "gsm8k",
    "score": 0.892,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'GSM8K'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "GSM8K",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "89.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "humaneval",
    "score": 0.713,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'HumanEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "HumanEval",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "71.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "ifeval-unspecified",
    "score": 0.902,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'IFEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "IFEval",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "90.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "infographicvqa",
    "score": 0.5,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'InfoVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "InfoVQA",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "livecodebench-unversioned",
    "score": 0.126,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'LiveCodeBench'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "LiveCodeBench",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "12.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "math-subset-unspecified",
    "score": 0.756,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'Math'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Math",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "75.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "mathvista-mini",
    "score": 0.5,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'MathVista (testmini)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MathVista (testmini)",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "mbpp",
    "score": 0.632,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'MBPP'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MBPP",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "63.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmlu-pro",
    "score": 0.436,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'MMLU (Pro)'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "MMLU (Pro)",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "43.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmmu",
    "score": 0.488,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'MMMU (val)'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "MMMU (val)",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "48.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "simpleqa",
    "score": 0.04,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'SimpleQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "section": "",
      "heading": "",
      "row_label": "SimpleQA",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 3,
      "raw_value": "4.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "textvqa",
    "score": 0.578,
    "source": "https://ai.google.dev/gemma/docs/core/model_card_3",
    "source_id": "gap-google-gemma3",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page for Gemma 3 (CC BY 4.0; 'Evaluation results marked with IT are for instruction-tuned models'), IT columns only (PT tables not used). n-shot as in the table's n-shot column. Page 'Last updated 2025-08-14 UTC'; saved copy hashed in evidence. Internal HiddenMath/Natural2Code/ECLeKTic, WMT24++ (metric unstated), BIG-Bench Extra Hard (aggregation unstated), AI2D and VQAv2 not used. Review date is not the evaluation-run date. Column 'Gemma 3 IT 4B', row 'TextVQA'.",
    "evidence": {
      "kind": "table",
      "table_index": 6,
      "section": "",
      "heading": "",
      "row_label": "TextVQA",
      "column_label": "Gemma 3 IT 4B",
      "column_index": 1,
      "raw_value": "57.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "aime-2025",
    "score": 0.116,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'AIME 2025'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "AIME 2025",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "11.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "codegolf-v2-2",
    "score": 0.168,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'Codegolf v2.2'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "Codegolf v2.2",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "16.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "global-mmlu",
    "score": 0.603,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'Global-MMLU'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "",
      "heading": "",
      "row_label": "Global-MMLU",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "60.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "gpqa-diamond-google-relaxed",
    "score": 0.237,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'GPQA Diamond'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "GPQA Diamond",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "23.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "humaneval",
    "score": 0.75,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'HumanEval'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "",
      "heading": "",
      "row_label": "HumanEval",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "75.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.257,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'LiveCodeBench v5'.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "",
      "heading": "",
      "row_label": "LiveCodeBench v5",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "25.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "mbpp",
    "score": 0.636,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'MBPP'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "",
      "heading": "",
      "row_label": "MBPP",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "63.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "mgsm",
    "score": 0.607,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'MGSM'.",
    "evidence": {
      "kind": "table",
      "table_index": 1,
      "section": "",
      "heading": "",
      "row_label": "MGSM",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "60.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "mmlu",
    "score": 0.649,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'MMLU'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "",
      "heading": "",
      "row_label": "MMLU",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "64.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "mmlu-pro",
    "score": 0.506,
    "source": "https://ai.google.dev/gemma/docs/gemma-3n/model_card",
    "source_id": "gap-google-gemma3n",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Google model card page (CC BY 4.0), E4B IT column; 0-shot unless stated; metric as in the Metric column. Same benchmark selection and ids as the existing E2B IT rows from this page (no PT, unversioned LCB, Global-MMLU-Lite, WMT ChrF, ECLeKTic, HiddenMath or MMLU-ProX rows). Page 'Last updated 2025-06-17 UTC'; saved copy hashed in evidence. Review date is not the evaluation-run date. Column 'E4B IT', row 'MMLU (Pro)'.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "",
      "heading": "",
      "row_label": "MMLU (Pro)",
      "column_label": "E4B IT",
      "column_index": 4,
      "raw_value": "50.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "emu3",
    "bench": "dpg-bench",
    "score": 0.806,
    "source": "https://arxiv.org/html/2409.18869v1",
    "source_id": "report-2409.18869v1",
    "by": "lab",
    "date": "2024-09-27",
    "mode": "text-to-image",
    "note": "BAAI's own evaluation of Emu3 (Emu3-Gen image generation); Emu3-DPO rows not used. Appendix Table 8 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "A2.T8",
      "caption": "Table 8: Comparison with state-of-the-art models on DPG-bench",
      "row_label": "Emu3",
      "column_label": "Overvall",
      "column_index": 1,
      "raw_value": "80.60",
      "scale_divisor": 100
    }
  },
  {
    "model": "emu3",
    "bench": "geneval",
    "score": 0.54,
    "source": "https://arxiv.org/html/2409.18869v1",
    "source_id": "report-2409.18869v1",
    "by": "lab",
    "date": "2024-09-27",
    "mode": "text-to-image",
    "note": "BAAI's own evaluation of Emu3 (Emu3-Gen image generation); Emu3-DPO rows not used. Appendix Table 7 GenEval overall without prompt rewriting (main-text Table 4 value 0.66 is marked as using rewriting).",
    "evidence": {
      "kind": "table",
      "table_id": "A2.T7",
      "caption": "Table 7: Comparison with state-of-the-art models on GenEval and T2I CompBench. Obj.: Object. Attri.: Attribute.",
      "row_label": "Emu3",
      "column_label": "GenEval / Overall",
      "column_index": 1,
      "raw_value": "0.54",
      "scale_divisor": 1
    }
  },
  {
    "model": "janus-1-3b",
    "bench": "geneval",
    "score": 0.61,
    "source": "https://arxiv.org/html/2410.13848v1",
    "source_id": "report-2410.13848v1",
    "by": "lab",
    "date": "2024-10-17",
    "mode": "text-to-image",
    "note": "DeepSeek-AI's own evaluation of Janus (1.3B). Table 3 GenEval overall.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Evaluation of text-to-image generation ability on GenEval benchmark. “Und.” and “Gen.” denote “understanding” and “generation”, respectively. Models using external pretrained diffusion model are marked with †.",
      "row_label": "Janus (Ours)",
      "column_label": "Overall \\uparrow",
      "column_index": 9,
      "raw_value": "0.61",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-4",
    "bench": "arena-hard-unspecified",
    "score": 0.754,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). Arena-Hard version/judge not stated (December 2024 report). (Benchmark id harmonised from arena-hard-auto-unspecified.)",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "ArenaHard",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "humaneval-plus",
    "score": 0.828,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). Second block; pass@1 as reported.",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "HumanEval+",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "ifeval-unspecified",
    "score": 0.63,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). IFEval aggregation not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "IFEval",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "63.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "livebench-unspecified",
    "score": 0.476,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). LiveBench release not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "LiveBench",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "47.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "mmlu-pro",
    "score": 0.704,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). Second (non-simple-evals) block; shots/decoding not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "MMLUPro",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4",
    "bench": "simple-evals-drop-f1",
    "score": 0.755,
    "source": "https://arxiv.org/html/2412.08905v1",
    "source_id": "report-2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "mode": "instruct",
    "note": "Microsoft's own evaluation, Table 1 phi-4 14b column (post-trained phi-4). Rows in the first block use OpenAI's simple-evals framework prompts/extraction with temperature 0.5 (caption). DROP scored with simple-evals (F1); caption places DROP in the simple-evals block.",
    "evidence": {
      "kind": "table",
      "table_id": "S1.T1",
      "caption": "Table 1: Performance of phi-4 on a set of standard benchmarks. The first set of benchmarks uses OpenAI’s simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.",
      "row_label": "DROP",
      "column_label": "Small models / phi-4 14b",
      "column_index": 2,
      "raw_value": "75.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "janus-pro-1b",
    "bench": "dpg-bench",
    "score": 0.8263,
    "source": "https://arxiv.org/html/2501.17811v1",
    "source_id": "report-2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "mode": "text-to-image",
    "note": "DeepSeek-AI's own evaluation of Janus-Pro-7B / Janus-Pro-1B text-to-image generation; understanding benchmarks not mapped to the image category. Table 5 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Performances on DPG-Bench. The methods in this table are all generation-specific models except Janus and Janus-Pro.",
      "row_label": "Janus-Pro-1B",
      "column_label": "Overall \\uparrow",
      "column_index": 6,
      "raw_value": "82.63",
      "scale_divisor": 100
    }
  },
  {
    "model": "janus-pro-1b",
    "bench": "geneval",
    "score": 0.73,
    "source": "https://arxiv.org/html/2501.17811v1",
    "source_id": "report-2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "mode": "text-to-image",
    "note": "DeepSeek-AI's own evaluation of Janus-Pro-7B / Janus-Pro-1B text-to-image generation; understanding benchmarks not mapped to the image category. Table 4 GenEval overall; not marked as using an external diffusion model or rewriter.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T4",
      "caption": "Table 4: Evaluation of text-to-image generation ability on GenEval benchmark. “Und.” and “Gen.” denote “understanding” and “generation”, respectively. Models using external pretrained diffusion model are marked with †.",
      "row_label": "Janus-Pro-1B",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.73",
      "scale_divisor": 1
    }
  },
  {
    "model": "janus-pro-7b",
    "bench": "dpg-bench",
    "score": 0.8419,
    "source": "https://arxiv.org/html/2501.17811v1",
    "source_id": "report-2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "mode": "text-to-image",
    "note": "DeepSeek-AI's own evaluation of Janus-Pro-7B / Janus-Pro-1B text-to-image generation; understanding benchmarks not mapped to the image category. Table 5 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Performances on DPG-Bench. The methods in this table are all generation-specific models except Janus and Janus-Pro.",
      "row_label": "Janus-Pro-7B",
      "column_label": "Overall \\uparrow",
      "column_index": 6,
      "raw_value": "84.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "janus-pro-7b",
    "bench": "geneval",
    "score": 0.8,
    "source": "https://arxiv.org/html/2501.17811v1",
    "source_id": "report-2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "mode": "text-to-image",
    "note": "DeepSeek-AI's own evaluation of Janus-Pro-7B / Janus-Pro-1B text-to-image generation; understanding benchmarks not mapped to the image category. Table 4 GenEval overall; not marked as using an external diffusion model or rewriter.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T4",
      "caption": "Table 4: Evaluation of text-to-image generation ability on GenEval benchmark. “Und.” and “Gen.” denote “understanding” and “generation”, respectively. Models using external pretrained diffusion model are marked with †.",
      "row_label": "Janus-Pro-7B",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.80",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "bfcl-version-unspecified",
    "score": 0.703,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot; BFCL version not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "caption": "Table 7: Phi-4-Mini language benchmark scores in comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma series.",
      "row_label": "BFCL (0-shot) [YMJ+24]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "bigcodebench-complete",
    "score": 0.43,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot; full vs hard subset not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T8",
      "caption": "Table 8: Phi-4-Mini coding performance comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma models.",
      "row_label": "BigCodeBench Completion (0-Shot) [ZVC+24]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "43.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "bigcodebench-instruct",
    "score": 0.338,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot; full vs hard subset not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T8",
      "caption": "Table 8: Phi-4-Mini coding performance comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma models.",
      "row_label": "BigCodeBench instruct (0-Shot) [ZVC+24]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "33.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "gpqa-diamond",
    "score": 0.369,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. Table 9 CoT reasoning evaluation, plain Phi-4-Mini row (reproduced by the authors, no asterisk); not the reasoning-enhanced rows.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T9",
      "caption": "Table 9: CoT Reasoning results of reasoning-enhanced Phi-4-Mini compared with larger 7B reasoning models and OpenAI models. An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.",
      "row_label": "Phi-4-Mini",
      "column_label": "GPQA Diamond",
      "column_index": 3,
      "raw_value": "36.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "humaneval",
    "score": 0.744,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot pass@1.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "caption": "Table 7: Phi-4-Mini language benchmark scores in comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma series.",
      "row_label": "HumanEval (0-Shot) [CTJ+21]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "humaneval-plus",
    "score": 0.683,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T8",
      "caption": "Table 8: Phi-4-Mini coding performance comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma models.",
      "row_label": "HumanEval+ (0-Shot) [LXWZ23]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "68.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "ifeval-unspecified",
    "score": 0.701,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 0-shot; IFEval aggregation not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "caption": "Table 7: Phi-4-Mini language benchmark scores in comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma series.",
      "row_label": "IFEval (0-shot) [ZLM+23]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "70.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "math-500",
    "score": 0.718,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. Table 9 CoT reasoning evaluation, plain Phi-4-Mini row (reproduced by the authors, no asterisk); not the reasoning-enhanced rows.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T9",
      "caption": "Table 9: CoT Reasoning results of reasoning-enhanced Phi-4-Mini compared with larger 7B reasoning models and OpenAI models. An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.",
      "row_label": "Phi-4-Mini",
      "column_label": "MATH-500",
      "column_index": 2,
      "raw_value": "71.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "mbpp",
    "score": 0.653,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 3-shot.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T7",
      "caption": "Table 7: Phi-4-Mini language benchmark scores in comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma series.",
      "row_label": "MBPP (3-Shot) [AON+21]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "65.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-mini",
    "bench": "mbpp-plus",
    "score": 0.638,
    "source": "https://arxiv.org/html/2503.01743v2",
    "source_id": "report-2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "mode": "instruct",
    "note": "Microsoft's own evaluation of Phi-4-Mini (3.8B, the released Phi-4-mini-instruct language model), 'Phi-4-Mini 3.8b' column; shots as printed in the row label. 3-shot.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T8",
      "caption": "Table 8: Phi-4-Mini coding performance comparison with Llama 3.2, Llama 3.1-8B, Qwen 2.5, Ministral and Gemma models.",
      "row_label": "MBPP+ (3-Shot) [LXWZ23]",
      "column_label": "Phi-4-Mini 3.8b",
      "column_index": 1,
      "raw_value": "63.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "ai2d-test",
    "score": 0.842,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "AI2D",
      "column_label": "12B",
      "column_index": 2,
      "raw_value": "84.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "bbeh-micro",
    "score": 0.163,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21). Recorded as bbeh-micro because the Gemma 4 report labels the identical Gemma 3 27B BBEH value (19.3) as the micro average.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "BBEH",
      "column_label": "Gemma 3 / 12B",
      "column_index": 7,
      "raw_value": "16.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmlu",
    "score": 0.719,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21).",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "MMLU",
      "column_label": "Gemma 3 / 12B",
      "column_index": 7,
      "raw_value": "71.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "ruler-128k",
    "score": 0.571,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 15, instruction-tuned (IT) column, RULER at 128K context.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T15",
      "caption": "Table 15: Performance of pre-trained (PT) and instruction fine-tuned (IT) models on long context benchmarks at different context lengths.",
      "row_label": "RULER / 128K",
      "column_label": "Gemma 3 IT / 12B",
      "column_index": 7,
      "raw_value": "57.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "vqav2-val",
    "score": 0.716,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "VQAv2 (val)",
      "column_label": "12B",
      "column_index": 2,
      "raw_value": "71.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-12b",
    "bench": "wmt24pp-chrf",
    "score": 0.516,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot, character-level F-score (chrF) per Table 21.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "WMT24++",
      "column_label": "Gemma 3 / 12B",
      "column_index": 7,
      "raw_value": "51.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "bbeh-micro",
    "score": 0.072,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21). Recorded as bbeh-micro because the Gemma 4 report labels the identical Gemma 3 27B BBEH value (19.3) as the micro average.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "BBEH",
      "column_label": "Gemma 3 / 1B",
      "column_index": 5,
      "raw_value": "7.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmlu",
    "score": 0.388,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21).",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "MMLU",
      "column_label": "Gemma 3 / 1B",
      "column_index": 5,
      "raw_value": "38.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-1b",
    "bench": "wmt24pp-chrf",
    "score": 0.359,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot, character-level F-score (chrF) per Table 21.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "WMT24++",
      "column_label": "Gemma 3 / 1B",
      "column_index": 5,
      "raw_value": "35.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "ai2d-test",
    "score": 0.845,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "AI2D",
      "column_label": "27B",
      "column_index": 3,
      "raw_value": "84.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmlu",
    "score": 0.769,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21).",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "MMLU",
      "column_label": "Gemma 3 / 27B",
      "column_index": 8,
      "raw_value": "76.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "ruler-128k",
    "score": 0.66,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 15, instruction-tuned (IT) column, RULER at 128K context.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T15",
      "caption": "Table 15: Performance of pre-trained (PT) and instruction fine-tuned (IT) models on long context benchmarks at different context lengths.",
      "row_label": "RULER / 128K",
      "column_label": "Gemma 3 IT / 27B",
      "column_index": 8,
      "raw_value": "66.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "vqav2-val",
    "score": 0.71,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "VQAv2 (val)",
      "column_label": "27B",
      "column_index": 3,
      "raw_value": "71.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-27b",
    "bench": "wmt24pp-chrf",
    "score": 0.534,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot, character-level F-score (chrF) per Table 21.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "WMT24++",
      "column_label": "Gemma 3 / 27B",
      "column_index": 8,
      "raw_value": "53.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "ai2d-test",
    "score": 0.748,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "AI2D",
      "column_label": "4B",
      "column_index": 1,
      "raw_value": "74.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "bbeh-micro",
    "score": 0.11,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21). Recorded as bbeh-micro because the Gemma 4 report labels the identical Gemma 3 27B BBEH value (19.3) as the micro average.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "BBEH",
      "column_label": "Gemma 3 / 4B",
      "column_index": 6,
      "raw_value": "11.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmlu",
    "score": 0.581,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot accuracy (Table 21).",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "MMLU",
      "column_label": "Gemma 3 / 4B",
      "column_index": 6,
      "raw_value": "58.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "ruler-128k",
    "score": 0.468,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 15, instruction-tuned (IT) column, RULER at 128K context.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T15",
      "caption": "Table 15: Performance of pre-trained (PT) and instruction fine-tuned (IT) models on long context benchmarks at different context lengths.",
      "row_label": "RULER / 128K",
      "column_label": "Gemma 3 IT / 4B",
      "column_index": 6,
      "raw_value": "46.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "vqav2-val",
    "score": 0.624,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 16 (IT models, multimodal); final test set with Pan&Scan unless the row names another split.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T16",
      "caption": "Table 16: Performance of instruction fine-tuned (IT) models on multimodal benchmarks. If not mentioned, these results are on the final test set of each dataset with P&S applied.",
      "row_label": "VQAv2 (val)",
      "column_label": "4B",
      "column_index": 1,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-3-4b",
    "bench": "wmt24pp-chrf",
    "score": 0.468,
    "source": "https://arxiv.org/html/2503.19786v1",
    "source_id": "report-2503.19786v1",
    "by": "lab",
    "date": "2025-03-25",
    "mode": "instruct",
    "note": "Google's own evaluation of the Gemma 3 instruction-tuned (IT) checkpoints; pre-trained (PT) tables are not used. Sampling, no normalisation (Table 21). Table 18 (IT models). 0-shot, character-level F-score (chrF) per Table 21.",
    "evidence": {
      "kind": "table",
      "table_id": "Sx1.T18",
      "caption": "Table 18: Performance of instruction fine-tuned (IT) models of different sizes on more internal and external benchmarks.",
      "row_label": "WMT24++",
      "column_label": "Gemma 3 / 4B",
      "column_index": 6,
      "raw_value": "46.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "aider-polyglot",
    "score": 0.147,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 13 code editing; internal reproduction with identical prompt.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 13",
      "caption": "Table 13: Code Editing Benchmarks. All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.",
      "row_label": "Command A",
      "column_label": "Aider Polyglot",
      "column_index": 2,
      "raw_value": "14.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "aime-2024",
    "score": 0.233,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 15; 2024 AIME.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 15",
      "caption": "Table 15: Reasoning performance of Command A compared to similarly-sized models. Benchmarks are MATH (Hendrycks et al., 2021), the 2024 AIME mathematics competition, and GPQA Diamond (Rein et al., 2023).",
      "row_label": "Command A",
      "column_label": "AIME (2024)",
      "column_index": 2,
      "raw_value": "23.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "bigcodebench-instruct",
    "score": 0.454,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 12; BigCodeBench Instruct (§4.4), 1-shot pass@1.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 12",
      "caption": "Table 12: Code Understanding Benchmarks across Python, Multi-language, and COBOL groups reporting 1-shot pass@1 and RepoQA reporting match accuracy. HE is HumanEval. All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduc",
      "row_label": "Command A",
      "column_label": "BigCodeBench",
      "column_index": 3,
      "raw_value": "45.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "bird-sql-dev",
    "score": 0.595,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 14; BIRD dev execution accuracy, internal reproduction.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 14",
      "caption": "Table 14: SQL Generation Benchmarks reporting execution accuracy against gold databases. All results are internal reproductions using an identical prompt.",
      "row_label": "Command A",
      "column_label": "Bird Dev",
      "column_index": 3,
      "raw_value": "59.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "ifeval-strict-mean",
    "score": 0.909,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 3; mean of prompt- and instruction-level strict accuracy (caption).",
    "evidence": {
      "kind": "table",
      "table_id": "Table 3",
      "caption": "Table 3: Results for Command A and Command R7B on standard academic benchmarks. * Note that for IFEval, Liu et al. (2024a) report only the prompt-level strict accuracy. We report the average of the prompt- and instruction-level strict accuracies for all other models (see Appendix B.7).",
      "row_label": "Command A",
      "column_label": "IFEval",
      "column_index": 4,
      "raw_value": "90.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "infobench",
    "score": 0.949,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 3.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 3",
      "caption": "Table 3: Results for Command A and Command R7B on standard academic benchmarks. * Note that for IFEval, Liu et al. (2024a) report only the prompt-level strict accuracy. We report the average of the prompt- and instruction-level strict accuracies for all other models (see Appendix B.7).",
      "row_label": "Command A",
      "column_label": "InFoBench",
      "column_index": 5,
      "raw_value": "94.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.269,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 12; LiveCodeBench Version 5, 10/24-2/25 (§4.4), 1-shot pass@1.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 12",
      "caption": "Table 12: Code Understanding Benchmarks across Python, Multi-language, and COBOL groups reporting 1-shot pass@1 and RepoQA reporting match accuracy. HE is HumanEval. All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduc",
      "row_label": "Command A",
      "column_label": "LiveCodeBench",
      "column_index": 2,
      "raw_value": "26.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "longbench-v2",
    "score": 0.434,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 22 LongBench-V2 overall.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 22",
      "caption": "Table 22: LongBench-V2 results for Command A.",
      "row_label": "Command A",
      "column_label": "Overall",
      "column_index": 1,
      "raw_value": "43.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "math",
    "score": 0.8,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 15; MATH (all).",
    "evidence": {
      "kind": "table",
      "table_id": "Table 15",
      "caption": "Table 15: Reasoning performance of Command A compared to similarly-sized models. Benchmarks are MATH (Hendrycks et al., 2021), the 2024 AIME mathematics competition, and GPQA Diamond (Rein et al., 2023).",
      "row_label": "Command A",
      "column_label": "MATH (all)",
      "column_index": 1,
      "raw_value": "80.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "mbpp-plus",
    "score": 0.862,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 12; 1-shot pass@1, internal reproduction with identical prompt.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 12",
      "caption": "Table 12: Code Understanding Benchmarks across Python, Multi-language, and COBOL groups reporting 1-shot pass@1 and RepoQA reporting match accuracy. HE is HumanEval. All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduc",
      "row_label": "Command A",
      "column_label": "MBPP+",
      "column_index": 1,
      "raw_value": "86.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "mmlu-pro",
    "score": 0.696,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 3; follows the simple-evals-style implementation per §4.1.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 3",
      "caption": "Table 3: Results for Command A and Command R7B on standard academic benchmarks. * Note that for IFEval, Liu et al. (2024a) report only the prompt-level strict accuracy. We report the average of the prompt- and instruction-level strict accuracies for all other models (see Appendix B.7).",
      "row_label": "Command A",
      "column_label": "MMLU-Pro",
      "column_index": 2,
      "raw_value": "69.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "repoqa",
    "score": 0.926,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 12; RepoQA 32K context, threshold 0.8 (§4.4).",
    "evidence": {
      "kind": "table",
      "table_id": "Table 12",
      "caption": "Table 12: Code Understanding Benchmarks across Python, Multi-language, and COBOL groups reporting 1-shot pass@1 and RepoQA reporting match accuracy. HE is HumanEval. All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduc",
      "row_label": "Command A",
      "column_label": "RepoQA",
      "column_index": 8,
      "raw_value": "92.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "ruler-128k",
    "score": 0.9,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 21 RULER at 128k.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 21",
      "caption": "Table 21: Results on the RULER long context benchmark. Not all models support contexts up to 256k.",
      "row_label": "Command A",
      "column_label": "128k",
      "column_index": 6,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.508,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 3 GPQA (simple-evals implementation per §4.1); Table 15 labels the same 50.8 value GPQA (Diamond).",
    "evidence": {
      "kind": "table",
      "table_id": "Table 3",
      "caption": "Table 3: Results for Command A and Command R7B on standard academic benchmarks. * Note that for IFEval, Liu et al. (2024a) report only the prompt-level strict accuracy. We report the average of the prompt- and instruction-level strict accuracies for all other models (see Appendix B.7).",
      "row_label": "Command A",
      "column_label": "GPQA",
      "column_index": 3,
      "raw_value": "50.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "simple-evals-mmlu",
    "score": 0.855,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 3; MMLU follows the simple-evals implementation (data, task and prompts) per §4.1.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 3",
      "caption": "Table 3: Results for Command A and Command R7B on standard academic benchmarks. * Note that for IFEval, Liu et al. (2024a) report only the prompt-level strict accuracy. We report the average of the prompt- and instruction-level strict accuracies for all other models (see Appendix B.7).",
      "row_label": "Command A",
      "column_label": "MMLU",
      "column_index": 1,
      "raw_value": "85.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "swe-bench-verified",
    "score": 0.268,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 13 code editing; internal reproduction with identical prompt; scaffold not stated in table.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 13",
      "caption": "Table 13: Code Editing Benchmarks. All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.",
      "row_label": "Command A",
      "column_label": "SWE-Bench Verified",
      "column_index": 1,
      "raw_value": "26.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "tau-bench-airline",
    "score": 0.453,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 6 original tau-bench setup, Pass@1 aggregated over 10 runs.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 6",
      "caption": "Table 6: Taubench Results. We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency; for example, Pass@4 is the probability that a model answers the same question correctly 4 times. Scores are aggregated over 10 runs.",
      "row_label": "Command A",
      "column_label": "Taubench Airline P@1",
      "column_index": 5,
      "raw_value": "45.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-a",
    "bench": "tau-bench-retail",
    "score": 0.6,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "source_id": "report-2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "mode": "instruct",
    "note": "Cohere's own evaluation of Command A (03-2025 weights); 'Command A' row only. Table 6 original tau-bench setup, Pass@1 aggregated over 10 runs.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 6",
      "caption": "Table 6: Taubench Results. We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency; for example, Pass@4 is the probability that a model answers the same question correctly 4 times. Scores are aggregated over 10 runs.",
      "row_label": "Command A",
      "column_label": "Taubench Retail P@1",
      "column_index": 1,
      "raw_value": "60.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "aime-2024",
    "score": 0.746,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). Table 1 average pass@1 (std in parentheses omitted).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T1",
      "caption": "Table 1: Average Pass@1 accuracy (%) of models on selected reasoning benchmarks. Bold denotes best model per benchmark and model class (i.e., open and closed model weights), and underline denotes the second best. We report standard deviation in parentheses for all results we produced using Eureka [1",
      "row_label": "Phi-4-reasoning",
      "column_label": "AIME 24",
      "column_index": 1,
      "raw_value": "74.6 (5.1)",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "aime-2025",
    "score": 0.631,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). Table 1 average pass@1.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T1",
      "caption": "Table 1: Average Pass@1 accuracy (%) of models on selected reasoning benchmarks. Bold denotes best model per benchmark and model class (i.e., open and closed model weights), and underline denotes the second best. We report standard deviation in parentheses for all results we produced using Eureka [1",
      "row_label": "Phi-4-reasoning",
      "column_label": "AIME 25",
      "column_index": 2,
      "raw_value": "63.1 (6.3)",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.671,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). GPQA Diamond (198 prompts per Table 4).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T1",
      "caption": "Table 1: Average Pass@1 accuracy (%) of models on selected reasoning benchmarks. Bold denotes best model per benchmark and model class (i.e., open and closed model weights), and underline denotes the second best. We report standard deviation in parentheses for all results we produced using Eureka [1",
      "row_label": "Phi-4-reasoning",
      "column_label": "GPQA-D",
      "column_index": 5,
      "raw_value": "67.1 (2.7)",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "hmmt-feb-2025",
    "score": 0.438,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). HMMT February 2025 (Table 1 sub-header and Table 4 dataset list).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T1",
      "caption": "Table 1: Average Pass@1 accuracy (%) of models on selected reasoning benchmarks. Bold denotes best model per benchmark and model class (i.e., open and closed model weights), and underline denotes the second best. We report standard deviation in parentheses for all results we produced using Eureka [1",
      "row_label": "Phi-4-reasoning",
      "column_label": "HMMT",
      "column_index": 3,
      "raw_value": "43.8 (6.2)",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "ifeval-unspecified",
    "score": 0.834,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). Table 2, averaged over five generations; labelled 'IFEval Strict' without prompt/instruction level.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T2",
      "caption": "Table 2: Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0. For ArenaHard \\star , ther",
      "row_label": "IFEval Strict [62]",
      "column_label": "Phi-4-reasoning",
      "column_index": 2,
      "raw_value": "83.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-4-reasoning",
    "bench": "livecodebench-202408-202501",
    "score": 0.538,
    "source": "https://arxiv.org/html/2504.21318v1",
    "source_id": "report-2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "mode": "reasoning",
    "note": "Microsoft's own evaluation of Phi-4-reasoning (not Phi-4-reasoning-plus): temperature 0.8, max 32,768 tokens (65,536 for AIME, HMMT, GPQA per Table 3). LiveCodeBench window 8/24-1/25 (Table 1 sub-header).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T1",
      "caption": "Table 1: Average Pass@1 accuracy (%) of models on selected reasoning benchmarks. Bold denotes best model per benchmark and model class (i.e., open and closed model weights), and underline denotes the second best. We report standard deviation in parentheses for all results we produced using Eureka [1",
      "row_label": "Phi-4-reasoning",
      "column_label": "LCB",
      "column_index": 6,
      "raw_value": "53.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "aime-2024",
    "score": 0.808,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). ",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "AIME24",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "\\textbf{80.8}\\;|\\;{20.0}",
      "part_index": 0,
      "part_value": "80.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "aime-2025",
    "score": 0.725,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). ",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "AIME25",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "\\textbf{72.5}\\;|\\;{16.7}",
      "part_index": 0,
      "part_value": "72.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "arena-hard-unspecified",
    "score": 0.87,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). Arena-Hard version/judge not stated. (Benchmark id harmonised from arena-hard-auto-unspecified.)",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "Arena Hard",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "87.0\\;|\\;{-}",
      "part_index": 0,
      "part_value": "87.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "bfcl-v2-live",
    "score": 0.741,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). ",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "BFCL V2 Live",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "74.1\\;|\\;{73.6}",
      "part_index": 0,
      "part_value": "74.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "gpqa-diamond",
    "score": 0.76,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). ",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "GPQA-Diamond",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "\\textbf{76.0}\\;|\\;{56.6}",
      "part_index": 0,
      "part_value": "76.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "ifeval-unspecified",
    "score": 0.889,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). Aggregation not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "IFEval",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "{88.9}\\;|\\;\\textbf{89.5}",
      "part_index": 0,
      "part_value": "88.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "livecodebench-202408-202502",
    "score": 0.663,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). Window 2408–2502 (the 2410–2502 row is not also imported).",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "LiveCodeBench (2408–2502)",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "\\textbf{66.3}\\;|\\;{29.0}",
      "part_index": 0,
      "part_value": "66.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "math-500",
    "score": 0.97,
    "source": "https://arxiv.org/html/2505.00949v5",
    "source_id": "report-2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "mode": "reasoning-on",
    "note": "NVIDIA's own evaluation, Table 5 'LN-Ultra Reasoning' column = Llama-3.1-Nemotron-Ultra-253B-v1 (not the SFT-only checkpoint). LN-Super (v1) is not the catalogue v1.5 checkpoint and is not used. Reasoning ON value (first of the on | off pair). ",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: LN-Ultra versus the strongest open-weight models, split by reasoning mode.",
      "row_label": "MATH500",
      "column_label": "LN-Ultra Reasoning / \\scriptsize\\textbf{{on}}\\;|\\;\\scriptsize\\textbf{{off}}",
      "column_index": 2,
      "raw_value": "97.0\\;|\\;{80.4}",
      "part_index": 0,
      "part_value": "97.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "bagel",
    "bench": "gedit-bench-en-overall",
    "score": 0.652,
    "source": "https://arxiv.org/html/2505.14683v3",
    "source_id": "report-2505.14683v3",
    "by": "lab",
    "date": "2025-07-27",
    "mode": "text-to-image",
    "note": "ByteDance Seed's own evaluation of BAGEL (7B active / 14B total MoT, the released BAGEL-7B-MoT). Table 7 GEdit-Bench-EN full set G_O (GPT-4.1), divided by 10.",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T7",
      "caption": "Table 7: Comparison on GEdit-Bench. All metrics are reported as higher-is-better ( \\uparrow ). G_SC, G_PQ, and G_O refer to the metrics evaluated by GPT-4.1.",
      "row_label": "BAGEL",
      "column_label": "GEdit-Bench-EN (Full set) \\uparrow / G_O",
      "column_index": 4,
      "raw_value": "6.52",
      "scale_divisor": 10
    }
  },
  {
    "model": "bagel",
    "bench": "geneval",
    "score": 0.82,
    "source": "https://arxiv.org/html/2505.14683v3",
    "source_id": "report-2505.14683v3",
    "by": "lab",
    "date": "2025-07-27",
    "mode": "text-to-image",
    "note": "ByteDance Seed's own evaluation of BAGEL (7B active / 14B total MoT, the released BAGEL-7B-MoT). Table 5 GenEval overall, plain BAGEL row (the BAGEL† row uses an LLM prompt rewriter and is not used).",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T5",
      "caption": "Table 5: Evaluation of text-to-image generation ability on GenEval benchmark. ‘Gen. Only’ stands for an image generation model, and ‘Unified’ denotes a model that has both understanding and generation capabilities. \\dagger refer to the methods using LLM rewriter.",
      "row_label": "BAGEL",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.82",
      "scale_divisor": 1
    }
  },
  {
    "model": "bagel",
    "bench": "wise-overall-publisher-run",
    "score": 0.7,
    "source": "https://arxiv.org/html/2505.14683v3",
    "source_id": "report-2505.14683v3",
    "by": "lab",
    "date": "2025-07-27",
    "mode": "self-cot",
    "note": "ByteDance Seed's own evaluation of BAGEL (7B active / 14B total MoT, the released BAGEL-7B-MoT). Table 6 WISE overall with BAGEL's own Self-CoT thinking (highest reasoning setting); plain BAGEL scores 0.52.",
    "evidence": {
      "kind": "table",
      "table_id": "S7.T6",
      "caption": "Table 6: Comparison of world knowledge reasoning on WISE. WISE examines the complex semantic understanding and world knowledge for T2I generation. ‘Gen. Only’ stands for an image generation model, and ‘Unified’ denotes a model that has both understanding and generation capabilities. **: Results of G",
      "row_label": "BAGEL w/ Self-CoT",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.70",
      "scale_divisor": 1
    }
  },
  {
    "model": "uniworld-v1",
    "bench": "gedit-bench-en-overall",
    "score": 0.485,
    "source": "https://arxiv.org/html/2506.03147v4",
    "source_id": "report-2506.03147v4",
    "by": "lab",
    "date": "2025-06-18",
    "mode": "text-to-image",
    "note": "The UniWorld-V1 authors' own evaluation of UniWorld-V1; competitor rows not used. WISE (0.55) is already recorded from the WISE source and is not repeated. Table 5 GEdit-Bench G_O (GPT-4.1), divided by 10; English set as used by GEdit-Bench-EN.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "caption": "Table 5: Comparison results on GEdit-Bench. G_SC, G_PQ, and G_O refer to the metrics evaluated by GPT-4.1.",
      "row_label": "UniWorld-V1",
      "column_label": "G_O \\uparrow",
      "column_index": 3,
      "raw_value": "4.85",
      "scale_divisor": 10
    }
  },
  {
    "model": "uniworld-v1",
    "bench": "geneval",
    "score": 0.8,
    "source": "https://arxiv.org/html/2506.03147v4",
    "source_id": "report-2506.03147v4",
    "by": "lab",
    "date": "2025-06-18",
    "mode": "text-to-image",
    "note": "The UniWorld-V1 authors' own evaluation of UniWorld-V1; competitor rows not used. WISE (0.55) is already recorded from the WISE source and is not repeated. Table 2 GenEval overall, plain UniWorld-V1 row (the UniWorld-V1† row uses an LLM rewriter and is not used).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T2",
      "caption": "Table 2: Comparison results on GenEval. “Gen. Only” refers to pure generation models, while “Unified” indicates models capable of both understanding and generation. \\dagger refers to the methods using LLM rewriter. {\\ddagger} : Results of GPT-4o-Image are tested by [47].",
      "row_label": "UniWorld-V1",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.80",
      "scale_divisor": 1
    }
  },
  {
    "model": "uniworld-v1",
    "bench": "imgedit-bench-overall",
    "score": 0.652,
    "source": "https://arxiv.org/html/2506.03147v4",
    "source_id": "report-2506.03147v4",
    "by": "lab",
    "date": "2025-06-18",
    "mode": "text-to-image",
    "note": "The UniWorld-V1 authors' own evaluation of UniWorld-V1; competitor rows not used. WISE (0.55) is already recorded from the WISE source and is not repeated. Table 4 ImgEdit-Bench overall, GPT-4.1 judge, divided by 5.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Comparison results on ImgEdit-Bench. “Overall” is calculated by averaging all scores across tasks. We use GPT-4.1 for evaluation.",
      "row_label": "UniWorld-V1",
      "column_label": "Overall \\uparrow",
      "column_index": 10,
      "raw_value": "3.26",
      "scale_divisor": 5
    }
  },
  {
    "model": "omnigen2",
    "bench": "gedit-bench-en-overall",
    "score": 0.721,
    "source": "https://arxiv.org/html/2506.18871v4",
    "source_id": "report-2506.18871v4",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "text-to-image",
    "note": "BAAI's own evaluation of OmniGen2 (v4 report). GenEval is deliberately not imported because v4 trains OmniGen2 with GenEval as an RL verifiable reward (§ RL setup). Table 3 GEdit-Bench-EN overall (O), divided by 10.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T3",
      "caption": "Table 3: Quantitative comparison on Emu-Edit [81] and GEdit-Bench-EN [54]. For Emu-Edit, CLIP-I/DINO measure consistency with the source image, while CLIP-Out measures alignment with the caption of target image, CLIP-B/32 [71] and DINO-S/16 [6] are leveraged for feature calculation. For GEdit-Bench,",
      "row_label": "OmniGen2",
      "column_label": "GEdit-Bench-EN / O \\uparrow",
      "column_index": 6,
      "raw_value": "7.21",
      "scale_divisor": 10
    }
  },
  {
    "model": "omnigen2",
    "bench": "oneig-en",
    "score": 0.47,
    "source": "https://arxiv.org/html/2506.18871v4",
    "source_id": "report-2506.18871v4",
    "by": "lab",
    "date": "2026-04-21",
    "mode": "text-to-image",
    "note": "BAAI's own evaluation of OmniGen2 (v4 report). GenEval is deliberately not imported because v4 trains OmniGen2 with GenEval as an RL verifiable reward (§ RL setup). Table 2 OneIG-Bench-EN overall.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T2",
      "caption": "Table 2: Comparison of different models across Understanding, Generation, Editing, and In-context Generation tasks. *: The first term represents the number of parameters for text generation, while the second term corresponds to the number of parameters allocated for image generation. \\dagger refers ",
      "row_label": "OmniGen2",
      "column_label": "Image Generation / OneIG-Bench-EN \\uparrow",
      "column_index": 6,
      "raw_value": "0.47",
      "scale_divisor": 1
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "chime4-wer",
    "score": 0.1059,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form C-4 column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / C-4",
      "column_index": 6,
      "raw_value": "10.59",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "earnings21-10m-wer",
    "score": 0.0962,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 long-form E21 10m column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Long-Form / E21 10m",
      "column_index": 8,
      "raw_value": "9.62",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "earnings22-10m-wer",
    "score": 0.1184,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 long-form E22 10m column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Long-Form / E22 10m",
      "column_index": 9,
      "raw_value": "11.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "gigaspeech-wer",
    "score": 0.1068,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form GS column. (Benchmark id harmonised from gigaspeech-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / GS",
      "column_index": 3,
      "raw_value": "10.68",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0186,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form LS-C column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / LS-C",
      "column_index": 1,
      "raw_value": "1.86",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0404,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form LS-O column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / LS-O",
      "column_index": 2,
      "raw_value": "4.04",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "spgispeech-wer",
    "score": 0.0219,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form SPGI column. (Benchmark id harmonised from spgispeech-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / SPGI",
      "column_index": 7,
      "raw_value": "2.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "switchboard-wer",
    "score": 0.1132,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form SB column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / SB",
      "column_index": 5,
      "raw_value": "11.32",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0685,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form VP column. (Benchmark id harmonised from voxpopuli-en-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Mini",
      "column_label": "Short-Form / VP",
      "column_index": 4,
      "raw_value": "6.85",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "chime4-wer",
    "score": 0.0964,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form C-4 column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / C-4",
      "column_index": 6,
      "raw_value": "9.64",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "earnings21-10m-wer",
    "score": 0.0955,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 long-form E21 10m column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Long-Form / E21 10m",
      "column_index": 8,
      "raw_value": "9.55",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "earnings22-10m-wer",
    "score": 0.1248,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 long-form E22 10m column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Long-Form / E22 10m",
      "column_index": 9,
      "raw_value": "12.48",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "gigaspeech-wer",
    "score": 0.1027,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form GS column. (Benchmark id harmonised from gigaspeech-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / GS",
      "column_index": 3,
      "raw_value": "10.27",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0153,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form LS-C column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / LS-C",
      "column_index": 1,
      "raw_value": "1.53",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0314,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form LS-O column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / LS-O",
      "column_index": 2,
      "raw_value": "3.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "spgispeech-wer",
    "score": 0.0189,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form SPGI column. (Benchmark id harmonised from spgispeech-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / SPGI",
      "column_index": 7,
      "raw_value": "1.89",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "switchboard-wer",
    "score": 0.1109,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form SB column.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / SB",
      "column_index": 5,
      "raw_value": "11.09",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0662,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's own evaluation of the open-weights Voxtral Mini / Voxtral Small (2507) checkpoints; the API-only 'Voxtral Mini Transcribe' row is not used. WER as a fraction (percent divided by 100). Table 3 short-form VP column. (Benchmark id harmonised from voxpopuli-en-test-wer.)",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task. We report short-form scores for LibriSpeech Test Clean (LS-C), LibriSpeech Test Other (LS-O), GigaSpeech (GS), VoxPopuli (VP), SwitchBoard (SB), CHiME-4 (C-4) and SPGISPeech (SPGI). We report long-form scores for Earnings-21 10m (E21 10m) and Earn",
      "row_label": "Voxtral Small",
      "column_label": "Short-Form / VP",
      "column_index": 4,
      "raw_value": "6.62",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "aime-2024",
    "score": 0.894,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; Avg@32 with LLM answer validation.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "AIME 24",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "89.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "bfcl-v3",
    "score": 0.764,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; Berkeley Function Calling Leaderboard V3.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "BFCL V3",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "76.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "browsecomp",
    "score": 0.213,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; context-management strategy not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "BrowseComp",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "21.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "gpqa-unspecified",
    "score": 0.75,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; Avg@8; the table labels the benchmark GPQA without naming the subset.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "GPQA",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "75.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "hle",
    "score": 0.106,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; text-only HLE questions, GPT-4o judge.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "HLE",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "10.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "ifeval-prompt-strict",
    "score": 0.863,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; IFEval prompt-level strict (text).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "IFEval",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "86.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "livecodebench-202407-202501",
    "score": 0.707,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; LiveCodeBench problems between 2024-07-01 and 2025-01-01 (footnote).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "LCB",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "70.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "math-500",
    "score": 0.981,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "MATH 500",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "98.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "mmlu",
    "score": 0.874,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; MMLU exact match.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "MMLU",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "87.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "mmlu-pro",
    "score": 0.814,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "MMLU Pro",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "81.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "multichallenge",
    "score": 0.425,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "MultiChallenge",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "42.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "scicode-unspecified",
    "score": 0.373,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; main-problem vs subproblem scoring not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "SciCode",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "37.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "simpleqa",
    "score": 0.145,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; SimpleQA correct rate.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "SimpleQA",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "14.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "swe-bench-verified",
    "score": 0.576,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 5; OpenHands v0.34.0, 100 iterations, history truncation, temperature 0.6, top_p 1.0.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "caption": "Table 5: Results on SWE-bench Verified and Terminal-Bench",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "57.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "sysbench-isr",
    "score": 0.774,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; SysBench instruction satisfaction rate (ISR).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "SysBench",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "77.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "tau-bench-airline",
    "score": 0.608,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; TAU-bench airline domain.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "TAU-Airline",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "60.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "tau-bench-retail",
    "score": 0.779,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; TAU-bench (original tau-bench) retail domain.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "TAU-Retail",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "77.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5-air",
    "bench": "terminal-bench-pre-2",
    "score": 0.3,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 5; Terminus framework with standard function calling; report predates Terminal-Bench 2.0.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "caption": "Table 5: Results on SWE-bench Verified and Terminal-Bench",
      "row_label": "Terminal-Bench",
      "column_label": "GLM- 4.5-Air",
      "column_index": 2,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2024",
    "score": 0.91,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; Avg@32 with LLM answer validation.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "AIME 24",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "91.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "bfcl-v3",
    "score": 0.778,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; Berkeley Function Calling Leaderboard V3.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "BFCL V3",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "77.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "browsecomp",
    "score": 0.264,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; context-management strategy not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "BrowseComp",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "26.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "gpqa-unspecified",
    "score": 0.791,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; Avg@8; the table labels the benchmark GPQA without naming the subset.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "GPQA",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "79.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "hle",
    "score": 0.144,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; text-only HLE questions, GPT-4o judge.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "HLE",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "14.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "ifeval-prompt-strict",
    "score": 0.861,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; IFEval prompt-level strict (text).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "IFEval",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "86.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "livecodebench-202407-202501",
    "score": 0.729,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; LiveCodeBench problems between 2024-07-01 and 2025-01-01 (footnote).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "LCB",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "72.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "math-500",
    "score": 0.982,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "MATH 500",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "98.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "mmlu",
    "score": 0.9,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; MMLU exact match.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "MMLU",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "mmlu-pro",
    "score": 0.846,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "MMLU Pro",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "84.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "multichallenge",
    "score": 0.528,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "MultiChallenge",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "52.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "scicode-unspecified",
    "score": 0.417,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 4 reasoning; main-problem vs subproblem scoring not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T4",
      "caption": "Table 4: Results on Reasoning Benchmarks. HLE represents Humanity’s Last Exam [28] and LCB represents LiveCodeBench (2407-2501) [19].",
      "row_label": "SciCode",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "41.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "simpleqa",
    "score": 0.264,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; SimpleQA correct rate.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "SimpleQA",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "26.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "swe-bench-verified",
    "score": 0.642,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 5; OpenHands v0.34.0, 100 iterations, history truncation, temperature 0.6, top_p 1.0.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "caption": "Table 5: Results on SWE-bench Verified and Terminal-Bench",
      "row_label": "SWE-bench Verified",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "64.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "sysbench-isr",
    "score": 0.81,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 6 general; SysBench instruction satisfaction rate (ISR).",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T6",
      "caption": "Table 6: Results on Commonly Used General Chat Benchmarks",
      "row_label": "SysBench",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "81.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "tau-bench-airline",
    "score": 0.604,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; TAU-bench airline domain.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "TAU-Airline",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "60.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "tau-bench-retail",
    "score": 0.797,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 3 agentic; TAU-bench (original tau-bench) retail domain.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T3",
      "caption": "Table 3: Results on Agentic Benchmarks. TAU represents TAU-bench [48] and BFCL represents Berkeley Function Calling Leaderboard [26].",
      "row_label": "TAU-Retail",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "79.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-5",
    "bench": "terminal-bench-pre-2",
    "score": 0.375,
    "source": "https://arxiv.org/html/2508.06471v1",
    "source_id": "report-2508.06471v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai/Zhipu's own evaluation of the post-trained GLM-4.5 / GLM-4.5-Air (hybrid reasoning models; the per-table thinking/non-thinking mode is not stated). Evaluation code: github.com/zai-org/glm-simple-evals. Base-model table and AA index not used. Table 5; Terminus framework with standard function calling; report predates Terminal-Bench 2.0.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T5",
      "caption": "Table 5: Results on SWE-bench Verified and Terminal-Bench",
      "row_label": "Terminal-Bench",
      "column_label": "GLM- 4.5",
      "column_index": 1,
      "raw_value": "37.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aider-polyglot",
    "score": 0.444,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Aider Polyglot",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "44.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aime-2024",
    "score": 0.958,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "AIME 2024 (no tools)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "95.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aime-2025",
    "score": 0.925,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "AIME 2025 (no tools)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "92.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "gpqa-diamond",
    "score": 0.801,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "GPQA Diamond (no tools)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "80.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "healthbench",
    "score": 0.576,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). HealthBench score (%).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HealthBench",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "57.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "healthbench-hard",
    "score": 0.3,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). HealthBench Hard score (%).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HealthBench Hard",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "30.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "hle",
    "score": 0.149,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HLE (no tools)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "14.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "hle-with-tools",
    "score": 0.19,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). With browsing/python tools as described in the model card.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HLE (with tools)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "19.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "mmlu",
    "score": 0.9,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "MMLU",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "90.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "mmmlu",
    "score": 0.813,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Average over the 14 MMMLU languages in Table 2.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "MMMLU (Average)",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "81.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "simpleqa",
    "score": 0.168,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "reasoning-level-not-stated",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Table 9 hallucination evaluation, accuracy metric, run without browsing; reasoning level not stated for this table.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T9",
      "caption": "Table 9: Hallucination evaluations",
      "row_label": "SimpleQA",
      "column_label": "gpt-oss-120b",
      "column_index": 2,
      "raw_value": "0.168",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "swe-bench-verified",
    "score": 0.624,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Coding evaluated with a Codex-CLI-like terminal tool.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "SWE-Bench Verified",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "tau-bench-airline",
    "score": 0.492,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Functions provided in the developer message.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Tau-Bench Airline",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "49.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "tau-bench-retail",
    "score": 0.678,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Functions provided in the developer message.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Tau-Bench Retail",
      "column_label": "gpt-oss-120b / high",
      "column_index": 3,
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aider-polyglot",
    "score": 0.342,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Aider Polyglot",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "34.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2024",
    "score": 0.921,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "AIME 2024 (no tools)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "92.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2025",
    "score": 0.917,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "AIME 2025 (no tools)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "91.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "gpqa-diamond",
    "score": 0.715,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "GPQA Diamond (no tools)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "71.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "healthbench",
    "score": 0.425,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). HealthBench score (%).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HealthBench",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "42.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "healthbench-hard",
    "score": 0.108,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). HealthBench Hard score (%).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HealthBench Hard",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "10.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "hle",
    "score": 0.109,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HLE (no tools)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "10.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "hle-with-tools",
    "score": 0.173,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). With browsing/python tools as described in the model card.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "HLE (with tools)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "17.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmlu",
    "score": 0.853,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6).",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "MMLU",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "85.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmmlu",
    "score": 0.757,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Average over the 14 MMMLU languages in Table 2.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "MMMLU (Average)",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "75.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "simpleqa",
    "score": 0.067,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "reasoning-level-not-stated",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Table 9 hallucination evaluation, accuracy metric, run without browsing; reasoning level not stated for this table.",
    "evidence": {
      "kind": "table",
      "table_id": "S4.T9",
      "caption": "Table 9: Hallucination evaluations",
      "row_label": "SimpleQA",
      "column_label": "gpt-oss-20b",
      "column_index": 3,
      "raw_value": "0.067",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "swe-bench-verified",
    "score": 0.607,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Coding evaluated with a Codex-CLI-like terminal tool.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "SWE-Bench Verified",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "60.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "tau-bench-airline",
    "score": 0.38,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Functions provided in the developer message.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Tau-Bench Airline",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "38.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-20b",
    "bench": "tau-bench-retail",
    "score": 0.548,
    "source": "https://arxiv.org/html/2508.10925v1",
    "source_id": "report-2508.10925v1",
    "by": "lab",
    "date": "2025-08-08",
    "mode": "high",
    "note": "OpenAI's own evaluation, Table 3 'high' reasoning-level column, pass@1 with the model's default system prompt (§2.6). Functions provided in the developer message.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T3",
      "caption": "Table 3: Evaluations across multiple benchmarks and reasoning levels.",
      "row_label": "Tau-Bench Retail",
      "column_label": "gpt-oss-20b / high",
      "column_index": 6,
      "raw_value": "54.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-base",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.8671,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 5 CVTG-2K average word accuracy (not NED/CLIPScore); header order Rank, Model, NED, CLIPScore, Word Accuracy 2/3/4/5 regions, average.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T5",
      "caption": "Table 5: Quantitative evaluation results of English text rendering on CVTG-2K Du et al. (2025).",
      "row_label": "Z-Image",
      "column_label": "Word Accuracy / average",
      "column_index": 8,
      "raw_value": "0.8671",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "dpg-bench",
    "score": 0.8814,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 10 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T10",
      "caption": "Table 10: Quantitative evaluation results on DPG Hu et al. (2024).",
      "row_label": "Z-Image",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "88.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-base",
    "bench": "geneval",
    "score": 0.84,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 9 GenEval overall; no prompt rewriter indicated.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T9",
      "caption": "Table 9: Quantitative Evaluation results on GenEval Ghosh et al. (2023).",
      "row_label": "Z-Image",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.84",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "longtext-cn",
    "score": 0.936,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 6 LongText-Bench-ZH.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T6",
      "caption": "Table 6: Quantitative evaluation results on LongText-Bench Geng et al. (2025).",
      "row_label": "Z-Image",
      "column_label": "LongText-Bench-ZH \\uparrow",
      "column_index": 3,
      "raw_value": "0.936",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "longtext-en",
    "score": 0.935,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 6 LongText-Bench-EN.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T6",
      "caption": "Table 6: Quantitative evaluation results on LongText-Bench Geng et al. (2025).",
      "row_label": "Z-Image",
      "column_label": "LongText-Bench-EN \\uparrow",
      "column_index": 2,
      "raw_value": "0.935",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "oneig-cn",
    "score": 0.535,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 8 OneIG-ZH overall (average of five dimensions).",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T8",
      "caption": "Table 8: Quantitative evaluation results on OneIG-ZH Chang et al. (2025). The overall score is the average of the five dimensions.",
      "row_label": "Z-Image",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.535",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "oneig-en",
    "score": 0.546,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 7 OneIG-EN overall (average of five dimensions).",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T7",
      "caption": "Table 7: Quantitative evaluation results on OneIG-EN Chang et al. (2025). The overall score is the average of the five dimensions.",
      "row_label": "Z-Image",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.546",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-base",
    "bench": "tiif-testmini-long",
    "score": 0.8304,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 11 TIIF-Bench testmini overall, long prompts, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T11",
      "caption": "Table 11: Quantitative evaluation results on TIIF Bench testmini Wei et al. (2025).",
      "row_label": "Z-Image",
      "column_label": "Overall / Avg / long",
      "column_index": 3,
      "raw_value": "83.04",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-base",
    "bench": "tiif-testmini-short",
    "score": 0.802,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 11 TIIF-Bench testmini overall, short prompts, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T11",
      "caption": "Table 11: Quantitative evaluation results on TIIF Bench testmini Wei et al. (2025).",
      "row_label": "Z-Image",
      "column_label": "Overall / Avg / short",
      "column_index": 2,
      "raw_value": "80.20",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.8585,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 5 CVTG-2K average word accuracy (not NED/CLIPScore); header order Rank, Model, NED, CLIPScore, Word Accuracy 2/3/4/5 regions, average.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T5",
      "caption": "Table 5: Quantitative evaluation results of English text rendering on CVTG-2K Du et al. (2025).",
      "row_label": "Z-Image-Turbo",
      "column_label": "Word Accuracy / average",
      "column_index": 8,
      "raw_value": "0.8585",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "dpg-bench",
    "score": 0.8486,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 10 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T10",
      "caption": "Table 10: Quantitative evaluation results on DPG Hu et al. (2024).",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "84.86",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "geneval",
    "score": 0.82,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 9 GenEval overall; no prompt rewriter indicated.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T9",
      "caption": "Table 9: Quantitative Evaluation results on GenEval Ghosh et al. (2023).",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall \\uparrow",
      "column_index": 8,
      "raw_value": "0.82",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "longtext-cn",
    "score": 0.926,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 6 LongText-Bench-ZH.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T6",
      "caption": "Table 6: Quantitative evaluation results on LongText-Bench Geng et al. (2025).",
      "row_label": "Z-Image-Turbo",
      "column_label": "LongText-Bench-ZH \\uparrow",
      "column_index": 3,
      "raw_value": "0.926",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "longtext-en",
    "score": 0.917,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 6 LongText-Bench-EN.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T6",
      "caption": "Table 6: Quantitative evaluation results on LongText-Bench Geng et al. (2025).",
      "row_label": "Z-Image-Turbo",
      "column_label": "LongText-Bench-EN \\uparrow",
      "column_index": 2,
      "raw_value": "0.917",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "oneig-cn",
    "score": 0.507,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 8 OneIG-ZH overall (average of five dimensions).",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T8",
      "caption": "Table 8: Quantitative evaluation results on OneIG-ZH Chang et al. (2025). The overall score is the average of the five dimensions.",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.507",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "oneig-en",
    "score": 0.528,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 7 OneIG-EN overall (average of five dimensions).",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T7",
      "caption": "Table 7: Quantitative evaluation results on OneIG-EN Chang et al. (2025). The overall score is the average of the five dimensions.",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.528",
      "scale_divisor": 1
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "tiif-testmini-long",
    "score": 0.8005,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 11 TIIF-Bench testmini overall, long prompts, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T11",
      "caption": "Table 11: Quantitative evaluation results on TIIF Bench testmini Wei et al. (2025).",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall / Avg / long",
      "column_index": 3,
      "raw_value": "80.05",
      "scale_divisor": 100
    }
  },
  {
    "model": "z-image-turbo",
    "bench": "tiif-testmini-short",
    "score": 0.7773,
    "source": "https://arxiv.org/html/2511.22699v5",
    "source_id": "report-2511.22699v5",
    "by": "lab",
    "date": "2026-07-06",
    "mode": "text-to-image",
    "note": "Alibaba Tongyi-MAI's own evaluation of Z-Image (the foundation/base checkpoint) and Z-Image-Turbo (distilled); Z-Image-Edit and competitor rows not used. Table 11 TIIF-Bench testmini overall, short prompts, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T11",
      "caption": "Table 11: Quantitative evaluation results on TIIF Bench testmini Wei et al. (2025).",
      "row_label": "Z-Image-Turbo",
      "column_label": "Overall / Avg / short",
      "column_index": 2,
      "raw_value": "77.73",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-image-edit",
    "bench": "gedit-bench-en-overall",
    "score": 0.764,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 10 GEdit-Bench-EN G_O, divided by 10.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T10",
      "caption": "Table 10: Performance comparison on GEdit-Bench.",
      "row_label": "LongCat-Image-Edit",
      "column_label": "GEdit-Bench-EN \\uparrow / G_O",
      "column_index": 3,
      "raw_value": "7.64",
      "scale_divisor": 10
    }
  },
  {
    "model": "longcat-image-edit",
    "bench": "imgedit-bench-overall",
    "score": 0.9,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 11 ImgEdit-Bench overall, divided by 5 (judge per ImgEdit protocol, not restated).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T11",
      "caption": "Table 11: Performance comparison on ImgEdit-Bench.",
      "row_label": "LongCat-Image-Edit",
      "column_label": "Overall \\uparrow",
      "column_index": 10,
      "raw_value": "4.50",
      "scale_divisor": 5
    }
  },
  {
    "model": "longcat-image",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.8658,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 6 CVTG-2K average word accuracy.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T6",
      "caption": "Table 6: Quantitative evaluation results of CVTG-2K.",
      "row_label": "LongCat-Image",
      "column_label": "Word Accuracy \\uparrow / average",
      "column_index": 5,
      "raw_value": "0.8658",
      "scale_divisor": 1
    }
  },
  {
    "model": "longcat-image",
    "bench": "dpg-bench",
    "score": 0.868,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 3 DPG-Bench overall, divided by 100.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T3",
      "caption": "Table 3: Quantitative evaluation results on DPG.",
      "row_label": "LongCat-Image",
      "column_label": "Overall \\uparrow",
      "column_index": 6,
      "raw_value": "86.80",
      "scale_divisor": 100
    }
  },
  {
    "model": "longcat-image",
    "bench": "geneval",
    "score": 0.87,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 2 GenEval overall.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T2",
      "caption": "Table 2: Quantitative evaluation results on GenEval.",
      "row_label": "LongCat-Image",
      "column_label": "Overall \\uparrow / Overall \\uparrow",
      "column_index": 7,
      "raw_value": "0.87",
      "scale_divisor": 1
    }
  },
  {
    "model": "longcat-image",
    "bench": "wise-overall-publisher-run",
    "score": 0.65,
    "source": "https://arxiv.org/html/2512.07584v1",
    "source_id": "report-2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "mode": "text-to-image",
    "note": "Meituan's own evaluation of LongCat-Image (text-to-image) and LongCat-Image-Edit (editing); competitor rows not used. Table 4 WISE overall; judge not restated in the table.",
    "evidence": {
      "kind": "table",
      "table_id": "S5.T4",
      "caption": "Table 4: Quantitative evaluation results of world knowledge reasoning on WISE.",
      "row_label": "LongCat-Image",
      "column_label": "Overall",
      "column_index": 7,
      "raw_value": "0.65",
      "scale_divisor": 1
    }
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "hle-with-tools",
    "score": 0.1548,
    "source": "https://arxiv.org/html/2512.20848v1",
    "source_id": "report-2512.20848v1",
    "by": "lab",
    "date": "2025-12-23",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 3 'N-3-Nano' column (post-trained Nemotron 3 Nano 30B-A3B); competitor columns excluded.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T3",
      "caption": "Table 3: Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B.",
      "row_label": "HLE (with tools)",
      "column_label": "N-3-Nano",
      "column_index": 1,
      "raw_value": "15.48",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.6825,
    "source": "https://arxiv.org/html/2512.20848v1",
    "source_id": "report-2512.20848v1",
    "by": "lab",
    "date": "2025-12-23",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 3 'N-3-Nano' column (post-trained Nemotron 3 Nano 30B-A3B); competitor columns excluded. LiveCodeBench v6, 2024-08 to 2025-05 window.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T3",
      "caption": "Table 3: Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B.",
      "row_label": "LiveCodeBench (v6 2024-08 \\leftrightarrow 2025-05)",
      "column_label": "N-3-Nano",
      "column_index": 1,
      "raw_value": "68.25",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "minif2f-pass1",
    "score": 0.5003,
    "source": "https://arxiv.org/html/2512.20848v1",
    "source_id": "report-2512.20848v1",
    "by": "lab",
    "date": "2025-12-23",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 3 'N-3-Nano' column (post-trained Nemotron 3 Nano 30B-A3B); competitor columns excluded.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T3",
      "caption": "Table 3: Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B.",
      "row_label": "MiniF2F pass@1",
      "column_label": "N-3-Nano",
      "column_index": 1,
      "raw_value": "50.03",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.595,
    "source": "https://arxiv.org/html/2512.20848v1",
    "source_id": "report-2512.20848v1",
    "by": "lab",
    "date": "2025-12-23",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 3 'N-3-Nano' column (post-trained Nemotron 3 Nano 30B-A3B); competitor columns excluded. Average over the report's MMLU-ProX languages.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T3",
      "caption": "Table 3: Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B.",
      "row_label": "MMLU-ProX (avg over langs)",
      "column_label": "N-3-Nano",
      "column_index": 1,
      "raw_value": "59.50",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "multichallenge",
    "score": 0.3845,
    "source": "https://arxiv.org/html/2512.20848v1",
    "source_id": "report-2512.20848v1",
    "by": "lab",
    "date": "2025-12-23",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 3 'N-3-Nano' column (post-trained Nemotron 3 Nano 30B-A3B); competitor columns excluded.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T3",
      "caption": "Table 3: Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B.",
      "row_label": "Scale AI Multi Challenge",
      "column_label": "N-3-Nano",
      "column_index": 1,
      "raw_value": "38.45",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp",
    "score": 0.52,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Without context management (the separate 'w/ Context Manage' row is not used).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "BrowseComp",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "52.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "gpqa-diamond",
    "score": 0.857,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "GPQA-Diamond",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "85.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "hle",
    "score": 0.248,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Text-only HLE subset, GPT-5.2 (medium) judge, max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HLE",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "24.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "hle-with-tools",
    "score": 0.428,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Text-only HLE subset with tools, GPT-5.2 (medium) judge, max 202,752 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HLE (w/ Tools)",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "42.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "hmmt-nov-2025",
    "score": 0.935,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HMMT Nov. 2025",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "93.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "imo-answerbench",
    "score": 0.82,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "IMO-AnswerBench",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "82.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "longbench-v2",
    "score": 0.591,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "LongBench v2",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "59.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-7",
    "bench": "toolathlon",
    "score": 0.238,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "Tool-Decathlon",
      "column_label": "GLM-4.7",
      "column_index": 2,
      "raw_value": "23.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "gpqa-diamond",
    "score": 0.86,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "GPQA-Diamond",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "hle",
    "score": 0.305,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Text-only HLE subset, GPT-5.2 (medium) judge, max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HLE",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "30.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "hle-with-tools",
    "score": 0.504,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Text-only HLE subset with tools, GPT-5.2 (medium) judge, max 202,752 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HLE (w/ Tools)",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "50.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "hmmt-feb-2025",
    "score": 0.979,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HMMT Feb. 2025",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "97.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "hmmt-nov-2025",
    "score": 0.969,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "HMMT Nov. 2025",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "96.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "imo-answerbench",
    "score": 0.825,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table. Max generation 131,072 tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "IMO-AnswerBench",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "82.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "longbench-v2",
    "score": 0.645,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "LongBench v2",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "64.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-5",
    "bench": "toolathlon",
    "score": 0.392,
    "source": "https://arxiv.org/html/2602.15763v2",
    "source_id": "report-2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "mode": "reasoning-default",
    "note": "Z.ai's own evaluation, Table 7, own GLM-5 and same-lab GLM-4.7 columns only (competitor columns excluded). Thinking/non-thinking mode is not labelled in the table.",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Comparison between GLM-5 and open-source/proprietary models. Results marked with * are from the full set of HLE. Results marked with † are evaluated on a verified version of Terminal-Bench 2.0, fixing some ambiguous instructions. The GDPval-AA Elo scores are recorded on 15th Feb., 2026. The",
      "row_label": "Tool-Decathlon",
      "column_label": "GLM-5",
      "column_index": 1,
      "raw_value": "39.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "imo-answerbench",
    "score": 0.793,
    "source": "https://arxiv.org/html/2603.19220v2",
    "source_id": "report-2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "mode": "thinking",
    "note": "NVIDIA's own evaluation, Table 1 'Nemotron-Cascade-2 30B-A3B' column only (no-tool value; bracketed TIR values not used); avg@8, 128K thinking budget, temperature 1.0, top_p 0.95 unless the report states otherwise.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T1",
      "caption": "Table 1: Main results. Nemotron-Cascade-2-30B-A3B achieves gold-medal performance in both the IMO 2025 and IOI 2025, which demonstrate remarkably high intelligence density. † Numbers in brackets refers to Tool-Integrated Reasoning (TIR) results. ‡ For the baseline models, we use official numbers whe",
      "row_label": "IMO AnswerBench",
      "column_label": "Nemotron-Cascade-2 30B-A3B",
      "column_index": 4,
      "raw_value": "79.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.725,
    "source": "https://arxiv.org/html/2603.19220v2",
    "source_id": "report-2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "mode": "thinking",
    "note": "NVIDIA's own evaluation, Table 1 'Nemotron-Cascade-2 30B-A3B' column only (no-tool value; bracketed TIR values not used); avg@8, 128K thinking budget, temperature 1.0, top_p 0.95 unless the report states otherwise.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T1",
      "caption": "Table 1: Main results. Nemotron-Cascade-2-30B-A3B achieves gold-medal performance in both the IMO 2025 and IOI 2025, which demonstrate remarkably high intelligence density. † Numbers in brackets refers to Tool-Integrated Reasoning (TIR) results. ‡ For the baseline models, we use official numbers whe",
      "row_label": "MMLU-ProX",
      "column_label": "Nemotron-Cascade-2 30B-A3B",
      "column_index": 4,
      "raw_value": "72.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "multichallenge",
    "score": 0.453,
    "source": "https://arxiv.org/html/2603.19220v2",
    "source_id": "report-2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "mode": "thinking",
    "note": "NVIDIA's own evaluation, Table 1 'Nemotron-Cascade-2 30B-A3B' column only (no-tool value; bracketed TIR values not used); avg@8, 128K thinking budget, temperature 1.0, top_p 0.95 unless the report states otherwise.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T1",
      "caption": "Table 1: Main results. Nemotron-Cascade-2-30B-A3B achieves gold-medal performance in both the IMO 2025 and IOI 2025, which demonstrate remarkably high intelligence density. † Numbers in brackets refers to Tool-Integrated Reasoning (TIR) results. ‡ For the baseline models, we use official numbers whe",
      "row_label": "Scale AI Multi-Challenge",
      "column_label": "Nemotron-Cascade-2 30B-A3B",
      "column_index": 4,
      "raw_value": "45.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "scicode-unspecified",
    "score": 0.364,
    "source": "https://arxiv.org/html/2603.19220v2",
    "source_id": "report-2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "mode": "thinking",
    "note": "NVIDIA's own evaluation, Table 1 'Nemotron-Cascade-2 30B-A3B' column only (no-tool value; bracketed TIR values not used); avg@8, 128K thinking budget, temperature 1.0, top_p 0.95 unless the report states otherwise. Table labels only 'SciCode'.",
    "evidence": {
      "kind": "table",
      "table_id": "S2.T1",
      "caption": "Table 1: Main results. Nemotron-Cascade-2-30B-A3B achieves gold-medal performance in both the IMO 2025 and IOI 2025, which demonstrate remarkably high intelligence density. † Numbers in brackets refers to Tool-Integrated Reasoning (TIR) results. ‡ For the baseline models, we use official numbers whe",
      "row_label": "SciCode",
      "column_label": "Nemotron-Cascade-2 30B-A3B",
      "column_index": 4,
      "raw_value": "36.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "browsecomp",
    "score": 0.3128,
    "source": "https://arxiv.org/html/2604.12374v1",
    "source_id": "report-2604.12374v1",
    "by": "lab",
    "date": "2026-04-14",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 5 'N-3-Super' column (BF16 Nemotron 3 Super 120B-A12B); competitor columns excluded. With search tool.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Evaluation suite for Nemotron 3 Super. We compare against Qwen-3.5-122B-A10B and GPT-OSS-120B.",
      "row_label": "BrowseComp with Search",
      "column_label": "N-3-Super",
      "column_index": 1,
      "raw_value": "31.28",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "hle-with-tools",
    "score": 0.2282,
    "source": "https://arxiv.org/html/2604.12374v1",
    "source_id": "report-2604.12374v1",
    "by": "lab",
    "date": "2026-04-14",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 5 'N-3-Super' column (BF16 Nemotron 3 Super 120B-A12B); competitor columns excluded.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Evaluation suite for Nemotron 3 Super. We compare against Qwen-3.5-122B-A10B and GPT-OSS-120B.",
      "row_label": "HLE (with tools)",
      "column_label": "N-3-Super",
      "column_index": 1,
      "raw_value": "22.82",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "mmlu-prox",
    "score": 0.7936,
    "source": "https://arxiv.org/html/2604.12374v1",
    "source_id": "report-2604.12374v1",
    "by": "lab",
    "date": "2026-04-14",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 5 'N-3-Super' column (BF16 Nemotron 3 Super 120B-A12B); competitor columns excluded. Average over the report's MMLU-ProX languages.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Evaluation suite for Nemotron 3 Super. We compare against Qwen-3.5-122B-A10B and GPT-OSS-120B.",
      "row_label": "MMLU-ProX (avg over langs)",
      "column_label": "N-3-Super",
      "column_index": 1,
      "raw_value": "79.36",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "multichallenge",
    "score": 0.5523,
    "source": "https://arxiv.org/html/2604.12374v1",
    "source_id": "report-2604.12374v1",
    "by": "lab",
    "date": "2026-04-14",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 5 'N-3-Super' column (BF16 Nemotron 3 Super 120B-A12B); competitor columns excluded.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T5",
      "caption": "Table 5: Evaluation suite for Nemotron 3 Super. We compare against Qwen-3.5-122B-A10B and GPT-OSS-120B.",
      "row_label": "Scale AI Multi-Challenge",
      "column_label": "N-3-Super",
      "column_index": 1,
      "raw_value": "55.23",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-m-1",
    "bench": "swe-bench-multilingual",
    "score": 0.631,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Agentic coding results for Laguna M.1 against open-weight MoE and dense baselines (Devstral 2, GLM-4.7, DeepSeek-V4-Flash, Qwen3.5-397B-A17B), with Claude Sonnet 4.6 shown as a frontier proprietary reference of comparable model size. “–” indicates a score not reported by the model provider.",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Laguna M.1",
      "column_index": 1,
      "raw_value": "63.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-m-1",
    "bench": "swe-bench-verified",
    "score": 0.746,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Agentic coding results for Laguna M.1 against open-weight MoE and dense baselines (Devstral 2, GLM-4.7, DeepSeek-V4-Flash, Qwen3.5-397B-A17B), with Claude Sonnet 4.6 shown as a frontier proprietary reference of comparable model size. “–” indicates a score not reported by the model provider.",
      "row_label": "SWE-bench Verified",
      "column_label": "Laguna M.1",
      "column_index": 1,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-m-1",
    "bench": "terminal-bench-2",
    "score": 0.458,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1). 32 CPUs / 48 GB RAM sandbox; some task images/verifiers patched for infrastructure reliability (Appendix A.3).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T7",
      "caption": "Table 7: Agentic coding results for Laguna M.1 against open-weight MoE and dense baselines (Devstral 2, GLM-4.7, DeepSeek-V4-Flash, Qwen3.5-397B-A17B), with Claude Sonnet 4.6 shown as a frontier proprietary reference of comparable model size. “–” indicates a score not reported by the model provider.",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "Laguna M.1",
      "column_index": 1,
      "raw_value": "45.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-multilingual",
    "score": 0.577,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T6",
      "caption": "Table 6: Agentic coding results for Laguna XS.2 against open-weight baselines of comparable size (Devstral Small 2, Gemma 4, Qwen3.5-35B-A3B, Qwen3.6-35B-A3B), with Claude Haiku 4.5 and GPT-5.4 Nano shown as frontier proprietary references of comparable model size. “–” indicates a score not reported",
      "row_label": "SWE-bench Multilingual",
      "column_label": "Laguna XS.2",
      "column_index": 1,
      "raw_value": "57.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-verified",
    "score": 0.699,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T6",
      "caption": "Table 6: Agentic coding results for Laguna XS.2 against open-weight baselines of comparable size (Devstral Small 2, Gemma 4, Qwen3.5-35B-A3B, Qwen3.6-35B-A3B), with Claude Haiku 4.5 and GPT-5.4 Nano shown as frontier proprietary references of comparable model size. “–” indicates a score not reported",
      "row_label": "SWE-bench Verified",
      "column_label": "Laguna XS.2",
      "column_index": 1,
      "raw_value": "69.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "laguna-xs-2",
    "bench": "terminal-bench-2",
    "score": 0.357,
    "source": "https://arxiv.org/html/2605.27605v1",
    "source_id": "report-2605.27605v1",
    "by": "lab",
    "date": "2026-05-26",
    "mode": "thinking",
    "note": "Poolside's own evaluation with its pool agent harness, max 500 steps, temperature 1.0, top_k 20, thinking mode, 256K context; mean pass@1 over four runs (§6.2.1). 32 CPUs / 48 GB RAM sandbox; some task images/verifiers patched for infrastructure reliability (Appendix A.3).",
    "evidence": {
      "kind": "table",
      "table_id": "S6.T6",
      "caption": "Table 6: Agentic coding results for Laguna XS.2 against open-weight baselines of comparable size (Devstral Small 2, Gemma 4, Qwen3.5-35B-A3B, Qwen3.6-35B-A3B), with Claude Haiku 4.5 and GPT-5.4 Nano shown as frontier proprietary references of comparable model size. “–” indicates a score not reported",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "Laguna XS.2",
      "column_index": 1,
      "raw_value": "35.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "browsecomp",
    "score": 0.444,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "BrowseComp",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "44.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "gpqa-unspecified",
    "score": 0.87,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. No tools; subset not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "GPQA (no tools)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "87.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "hle",
    "score": 0.267,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "HLE (no tools)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "26.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "hle-with-tools",
    "score": 0.374,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "HLE (with tools)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "37.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "ifbench-prompt-loose",
    "score": 0.817,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "IFBench (prompt loose)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "81.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "imo-answerbench",
    "score": 0.886,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. No tools.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "IMOAnswerBench (no tools)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "88.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.89,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. v6, date window not stated.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "LiveCodeBench (v6)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "89.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "longbench-v2",
    "score": 0.619,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. LongBench v2 items up to 1M tokens.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "Longbench v2 ( \\leq 1M)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "61.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "mmlu-pro",
    "score": 0.868,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "MMLU-Pro",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "86.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "mmlu-prox",
    "score": 0.83,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. Average over en/de/fr/es/it/ja/zh/hi/pt/ko.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "83.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "multichallenge",
    "score": 0.638,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "Multi-Challenge",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "63.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "scicode-subtask",
    "score": 0.446,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "SciCode (subtask)",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "44.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "swe-bench-multilingual",
    "score": 0.677,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "SWE-Bench Multilingual",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "67.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "swe-bench-verified",
    "score": 0.707,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "SWE-Bench Verified",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "70.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "tau3-bench",
    "score": 0.709,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations. TauBench V3 average over Airline, Retail, Telecom and Banking (components not imported).",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "Average",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "70.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "terminal-bench-2.1",
    "score": 0.564,
    "source": "https://arxiv.org/html/2606.15007v1",
    "source_id": "report-2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "mode": "reasoning",
    "note": "NVIDIA's own evaluation, Table 10 'N-3-Ultra 550B-A55B' column (BF16 checkpoint). All models run under the same NVIDIA evaluation settings; inference parameters from each model card's recommendations.",
    "evidence": {
      "kind": "table",
      "table_id": "S3.T10",
      "caption": "Table 10: Evaluation suite for Nemotron 3 Ultra. We compare against six open models.",
      "row_label": "Terminal Bench 2.1",
      "column_label": "N-3-Ultra 550B-A55B",
      "column_index": 1,
      "raw_value": "56.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "aime-2026",
    "score": 0.947,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'AIME 2026', section '*General Capabilities and Reasoning*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Capabilities and Reasoning*",
      "heading": "Benchmarks",
      "row_label": "AIME 2026",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "94.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "charxiv-rq",
    "score": 0.788,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'Charxiv Reasoning', section '*Multimodal*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*Multimodal*",
      "heading": "Benchmarks",
      "row_label": "Charxiv Reasoning",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "78.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "deepsearchqa",
    "score": 0.746,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'DeepSearch QA', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "DeepSearch QA",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "gaia2",
    "score": 0.433,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'Gaia2', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "Gaia2",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "ifbench-unspecified",
    "score": 0.77,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'IFBench', section '*General Capabilities and Reasoning*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Capabilities and Reasoning*",
      "heading": "Benchmarks",
      "row_label": "IFBench",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "77.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "mcp-atlas",
    "score": 0.755,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'MCP Atlas (Public)', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "MCP Atlas (Public)",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "75.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.758,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'OmniDocBench v1.5', section '*Multimodal*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*Multimodal*",
      "heading": "Benchmarks",
      "row_label": "OmniDocBench v1.5",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "75.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "osworld-verified",
    "score": 0.659,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'OSWorld-Verified', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "OSWorld-Verified",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "65.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "screenspot-pro",
    "score": 0.754,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'ScreenSpot Pro', section '*Multimodal*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*Multimodal*",
      "heading": "Benchmarks",
      "row_label": "ScreenSpot Pro",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "skillsbench-with-skills",
    "score": 0.443,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'SkillsBench (with skills)', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "SkillsBench (with skills)",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "44.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.512,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'SWE-Bench Pro', section '*Agentic Coding*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*Agentic Coding*",
      "heading": "Benchmarks",
      "row_label": "SWE-Bench Pro",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "51.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "swe-bench-verified",
    "score": 0.76,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'SWE-Bench Verified', section '*Agentic Coding*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*Agentic Coding*",
      "heading": "Benchmarks",
      "row_label": "SWE-Bench Verified",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "76.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "wildclawbench",
    "score": 0.476,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "source_id": "speed-meta-muse-glimmer-30b",
    "by": "lab",
    "date": "2026-08-11",
    "mode": "high reasoning",
    "note": "Meta Superintelligence Lab own evaluation in the pinned Muse Glimmer model card, Benchmarks table, 'Muse Glimmer-30B High Reasoning' column (high reasoning strength; recommended sampling temperature=1.0, top_p=0.95, top_k=64). Rows labelled (AA), AA-LCR, GDPVal-AA, Beam128K, the dual-metric security rows and the Preparedness table are not used. Column 'Muse Glimmer-30B High Reasoning', row 'WildClawBench', section '*General Agentic*'.",
    "evidence": {
      "kind": "table",
      "table_index": 4,
      "section": "*General Agentic*",
      "heading": "Benchmarks",
      "row_label": "WildClawBench",
      "column_label": "Muse Glimmer-30B High Reasoning",
      "column_index": 2,
      "raw_value": "47.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "ernie-image-turbo",
    "bench": "longtext-cn",
    "score": 0.9675,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "LongTextBench",
      "row_label": "ERNIE-Image-Turbo (w/o PE)",
      "column_label": "LongText-Bench-ZH",
      "column_index": 2,
      "raw_value": "0.9675",
      "scale_divisor": 1
    }
  },
  {
    "model": "ernie-image-turbo",
    "bench": "longtext-en",
    "score": 0.9602,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "LongTextBench",
      "row_label": "ERNIE-Image-Turbo (w/o PE)",
      "column_label": "LongText-Bench-EN",
      "column_index": 1,
      "raw_value": "0.9602",
      "scale_divisor": 1
    }
  },
  {
    "model": "ernie-image-turbo",
    "bench": "oneig-cn",
    "score": 0.5062,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional. OneIG-ZH (Chinese split) Overall.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "OneIG-ZH",
      "row_label": "ERNIE-Image-Turbo (w/o PE)",
      "column_label": "Overall",
      "column_index": 6,
      "raw_value": "0.5062",
      "scale_divisor": 1
    }
  },
  {
    "model": "ernie-image",
    "bench": "longtext-cn",
    "score": 0.9594,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "LongTextBench",
      "row_label": "ERNIE-Image (w/o PE)",
      "column_label": "LongText-Bench-ZH",
      "column_index": 2,
      "raw_value": "0.9594",
      "scale_divisor": 1
    }
  },
  {
    "model": "ernie-image",
    "bench": "longtext-en",
    "score": 0.9679,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional.",
    "evidence": {
      "kind": "table",
      "table_index": 3,
      "section": "LongTextBench",
      "row_label": "ERNIE-Image (w/o PE)",
      "column_label": "LongText-Bench-EN",
      "column_index": 1,
      "raw_value": "0.9679",
      "scale_divisor": 1
    }
  },
  {
    "model": "ernie-image",
    "bench": "oneig-cn",
    "score": 0.5208,
    "source": "https://huggingface.co/baidu/ERNIE-Image/blob/5346b31d68c9c23758ba56ef8be5e9dc174c7f99/README.md",
    "source_id": "visual-ernie-image",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "w/o PE",
    "note": "Baidu's own evaluation in the pinned ERNIE-Image card, row without prompt enhancer (w/o PE), matching the existing ERNIE-Image rows from this source. Already fractional. OneIG-ZH (Chinese split) Overall.",
    "evidence": {
      "kind": "table",
      "table_index": 2,
      "section": "OneIG-ZH",
      "row_label": "ERNIE-Image (w/o PE)",
      "column_label": "Overall",
      "column_index": 6,
      "raw_value": "0.5208",
      "scale_divisor": 1
    }
  },
  {
    "model": "glm-4-6",
    "bench": "aime-2025-with-tools",
    "score": 0.986,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Tool-integrated reasoning ('w/ Tools'); the no-tool 93.9 is already recorded. Printed label '98.6' for 'AIME 25'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "AIME 25",
      "column_label": "GLM-4.6 (w/ Tools label above bar)",
      "raw_value": "98.6",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-6",
    "bench": "gpqa-unspecified-with-tools",
    "score": 0.829,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Tool-integrated reasoning ('w/ Tools'); chart labels only 'GPQA'; the no-tool 81.0 is already recorded as gpqa-diamond. Printed label '82.9' for 'GPQA'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "GPQA",
      "column_label": "GLM-4.6 (w/ Tools label above bar)",
      "raw_value": "82.9",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-6",
    "bench": "livecodebench-v6-window-unspecified-with-tools",
    "score": 0.845,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Tool-integrated reasoning ('w/ Tools'); v6 window not stated; the no-tool 82.8 is already recorded. Printed label '84.5' for 'LiveCodeBench v6'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "LiveCodeBench v6",
      "column_label": "GLM-4.6 (w/ Tools label above bar)",
      "raw_value": "84.5",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-6",
    "bench": "terminal-bench-pre-2",
    "score": 0.405,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Chart published 2025-09-30, before Terminal-Bench 2.0; task-set version and harness not stated (GLM-4.5's 37.5 in the same chart equals its terminal-bench-pre-2 row from the GLM-4.5 report). Printed label '40.5' for 'Terminal-Bench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "Terminal-Bench",
      "column_label": "GLM-4.6",
      "raw_value": "40.5",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-6",
    "bench": "tau2-bench-weighted",
    "score": 0.759,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Label '(Weighted)'; weighting not spelled out. Differs from the 75.2 simple-average tau2-bench row in the GLM-4.7 card. Printed label '75.9' for 'τ²-Bench (Weighted)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "τ²-Bench (Weighted)",
      "column_label": "GLM-4.6",
      "raw_value": "75.9",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2025",
    "score": 0.854,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. No tool label on this bar. Printed label '85.4' for 'AIME 25'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "AIME 25",
      "column_label": "GLM-4.5",
      "raw_value": "85.4",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-5",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.633,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. v6 window not stated; separate from the 2407-2501 row in the GLM-4.5 report. Printed label '63.3' for 'LiveCodeBench v6'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "LiveCodeBench v6",
      "column_label": "GLM-4.5",
      "raw_value": "63.3",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-4-5",
    "bench": "tau2-bench-weighted",
    "score": 0.675,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "source_id": "card2-zai-org-glm-4-6",
    "by": "lab",
    "date": "2025-09-30",
    "mode": "hybrid-mode-not-stated",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-4.5, committed in 6fb1bbf on 2025-09-30 before the README revision and removed from main on 2025-12-22, so the README embed now returns 404; archived from the pinned commit and hashed in evidence; visually read). GLM-4.6 (blue) and same-lab GLM-4.5 (green) columns only; DeepSeek-V3.2-Exp and Claude columns not used. Chart subtitle: results under 128K context length. README recommends temperature 1.0 (top_p 0.95, top_k 40 for code tasks); thinking/non-thinking mode not stated. Cells already recorded for these models (GLM-4.7 card, GLM-4.5 report) are not repeated. Label '(Weighted)'; weighting not spelled out. Printed label '67.5' for 'τ²-Bench (Weighted)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_glm46.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png)",
      "row_label": "τ²-Bench (Weighted)",
      "column_label": "GLM-4.5",
      "raw_value": "67.5",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6/resources/bench_glm46.png",
      "asset_sha256": "fe12649f14841567f9a7f1aa10a200fed331f7a59dc992583e4f2eacf07c0c46",
      "asset_file": "evidence/zai-org__GLM-4.5-github@6fb1bbf4e4a5d08f5e8269cd6b87660e56397ff6.resources__bench_glm46.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-4.5/refs/heads/main/resources/bench_glm46.png"
    }
  },
  {
    "model": "glm-5-3-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.843,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "max reasoning effort",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: evaluated in Claude Code 2.1.207, temperature 1.0, top_p 1, max_new_tokens 65536, 6 h timeout. The same value is in .eval_results/GLM-5.3-Flash.yaml at the pinned revision. Printed label '84.3' for 'Terminal Bench 2.1'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "Terminal Bench 2.1",
      "column_label": "GLM-5.3-Flash",
      "raw_value": "84.3",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-3-flash",
    "bench": "deepswe-v1-1",
    "score": 0.634,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "max reasoning effort",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: mini-swe-agent harness, temperature 0.95, top_p 1.0, 6 h timeout, 400K context. The same value is in .eval_results/GLM-5.3-Flash.yaml at the pinned revision. Printed label '63.4' for 'DeepSWE v1.1'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "DeepSWE v1.1",
      "column_label": "GLM-5.3-Flash",
      "raw_value": "63.4",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-3-flash",
    "bench": "agents-last-exam-unspecified",
    "score": 0.263,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "max reasoning effort",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. The card's footnote for Agents' Last Exam is blank, so metric and harness are unknown. Printed label '26.3' for 'Agents' Last Exam'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "Agents' Last Exam",
      "column_label": "GLM-5.3-Flash",
      "raw_value": "26.3",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-3-flash",
    "bench": "automationbench-v1-0-6",
    "score": 0.488,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "max reasoning effort",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: AutomationBench v1.0.6 with the null-type handling fix from zapier/AutomationBench PR #13. Printed label '48.8' for 'AutomationBench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "AutomationBench",
      "column_label": "GLM-5.3-Flash",
      "raw_value": "48.8",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-3-flash",
    "bench": "hle-with-tools",
    "score": 0.553,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "max reasoning effort",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: HLE w/ tools on the full set, temperature 1.0, top_p 0.95, max generation 163,840 tokens, 300K context with context management, GPT-5.6-luna (medium) judge. The same value is in .eval_results/GLM-5.3-Flash.yaml at the pinned revision. Printed label '55.3' for 'HLE w/ Tools'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "HLE w/ Tools",
      "column_label": "GLM-5.3-Flash",
      "raw_value": "55.3",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-2",
    "bench": "deepswe-v1-1",
    "score": 0.462,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "default (not stated)",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: mini-swe-agent harness, temperature 0.95, top_p 1.0, 6 h timeout, 400K context. Printed label '46.2' for 'DeepSWE v1.1'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "DeepSWE v1.1",
      "column_label": "GLM-5.2",
      "raw_value": "46.2",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-2",
    "bench": "agents-last-exam-unspecified",
    "score": 0.204,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "default (not stated)",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. The card's footnote for Agents' Last Exam is blank, so metric and harness are unknown. Printed label '20.4' for 'Agents' Last Exam'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "Agents' Last Exam",
      "column_label": "GLM-5.2",
      "raw_value": "20.4",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "glm-5-2",
    "bench": "automationbench-v1-0-6",
    "score": 0.262,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "source_id": "card2-zai-org-glm-5-3-flash",
    "by": "lab",
    "date": "2026-09-07",
    "mode": "default (not stated)",
    "note": "Z.ai own results in the benchmark chart embedded by the pinned zai-org/GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png in zai-org's Apache-2.0 GitHub repository zai-org/GLM-5, committed in f6adb53 on 2026-08-26 and unchanged on main; archived copy hashed in evidence; visually read). GLM-5.3-Flash (blue) and same-lab GLM-5.2 (green) columns only; DeepSeek-V4-Vision-Exp, Claude, GPT and Gemini columns not used. GLM-5.3-Flash reasoning_effort defaults to max, which the card says to keep for benchmark reproduction; GLM-5.2's mode is not restated. GDPval-AA v2 (Artificial Analysis Elo) not used. Card footnote: AutomationBench v1.0.6 with the null-type handling fix from zapier/AutomationBench PR #13. Printed label '26.2' for 'AutomationBench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "resources/bench_53.png (embedded by the README as https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png)",
      "row_label": "AutomationBench",
      "column_label": "GLM-5.2",
      "raw_value": "26.2",
      "scale_divisor": 100,
      "asset_url": "https://raw.githubusercontent.com/zai-org/GLM-5/f6adb53af3c1c620f5a315c127c8875774ff6196/resources/bench_53.png",
      "asset_sha256": "ea9de7784afc1e3ce4d72fac927e683043823307042aa2f4ae616357074b372a",
      "asset_file": "evidence/zai-org__GLM-5-github@f6adb53af3c1c620f5a315c127c8875774ff6196.resources__bench_53.png",
      "embedded_url": "https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "gpqa-diamond",
    "score": 0.712,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. Shots/CoT not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '71.2' for 'GPQA Diamond'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "GPQA Diamond",
      "column_label": "Mistral Small 4 (Reasoning)",
      "raw_value": "71.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "mmlu-pro",
    "score": 0.78,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. Shots/CoT not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '78' for 'MMLU Pro'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "MMLU Pro",
      "column_label": "Mistral Small 4 (Reasoning)",
      "raw_value": "78",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "ifbench-unspecified",
    "score": 0.48,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. Aggregation not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '48' for 'AllenAI IFBench'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "AllenAI IFBench",
      "column_label": "Mistral Small 4 (Reasoning)",
      "raw_value": "48",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "arena-hard-unspecified",
    "score": 0.583,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. Version and judge not labelled (the Small 3.2 bar, 43.1, equals its card's Arena Hard v2 value, but v2 is not asserted). Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '58.3' for 'Arena Hard'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "Arena Hard",
      "column_label": "Mistral Small 4 (Reasoning)",
      "raw_value": "58.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "mmmu-pro",
    "score": 0.6,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. Vision benchmark; setting not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '60' for 'MMMU-Pro'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "MMMU-Pro",
      "column_label": "Mistral Small 4 (Reasoning)",
      "raw_value": "60",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-4",
    "bench": "collie",
    "score": 0.629,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "high",
    "note": "Mistral own results in the pinned repository chart images/image3.png ('Performance Comparison Across Internal Models', embedded under README 'Comparing Reasoning Models'; visually read; archived copy hashed in evidence). Mistral Small 4 bar: bold top label of the hatched Reasoning segment (reasoning_effort=high; the README's only reasoning setting); the inner Instruct label is not used. COLLIE; subset and aggregation not stated. Magistral Medium 1.2 is not in the catalogue; AA LCR (custom implementation) not used. Printed label '62.9' for 'Collie'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image3.png",
      "row_label": "Collie",
      "column_label": "Mistral Small 4 - High",
      "raw_value": "62.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image3.png",
      "asset_sha256": "b38c8a96b124ff831fa9e7270939123565b396485b48f3219f0387142661b735",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image3.png"
    }
  },
  {
    "model": "mistral-large-3",
    "bench": "gpqa-diamond",
    "score": 0.641,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Large 3' (single Instruct bar; the catalogue's only Mistral Large 3 instruct checkpoint, 2512). Shots/CoT not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '64.1' for 'GPQA Diamond'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "GPQA Diamond",
      "column_label": "Mistral Large 3",
      "raw_value": "64.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-large-3",
    "bench": "mmlu-pro",
    "score": 0.809,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Large 3' (single Instruct bar; the catalogue's only Mistral Large 3 instruct checkpoint, 2512). Shots/CoT not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '80.9' for 'MMLU Pro'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "MMLU Pro",
      "column_label": "Mistral Large 3",
      "raw_value": "80.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-large-3",
    "bench": "ifbench-unspecified",
    "score": 0.378,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Large 3' (single Instruct bar; the catalogue's only Mistral Large 3 instruct checkpoint, 2512). Aggregation not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '37.8' for 'AllenAI IFBench'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "AllenAI IFBench",
      "column_label": "Mistral Large 3",
      "raw_value": "37.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-large-3",
    "bench": "arena-hard-unspecified",
    "score": 0.667,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Large 3' (single Instruct bar; the catalogue's only Mistral Large 3 instruct checkpoint, 2512). Version and judge not labelled (the Small 3.2 bar, 43.1, equals its card's Arena Hard v2 value, but v2 is not asserted). Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '66.7' for 'Arena Hard'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "Arena Hard",
      "column_label": "Mistral Large 3",
      "raw_value": "66.7",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-large-3",
    "bench": "mmmu-pro",
    "score": 0.54,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Large 3' (single Instruct bar; the catalogue's only Mistral Large 3 instruct checkpoint, 2512). Vision benchmark; setting not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '54' for 'MMMU-Pro'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "MMMU-Pro",
      "column_label": "Mistral Large 3",
      "raw_value": "54",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "ifbench-unspecified",
    "score": 0.34,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Small 3.2' (single Instruct bar; Mistral-Small-3.2-24B-Instruct-2506). Its GPQA Diamond 50, MMLU Pro 69 and Arena Hard 43.1 are not added: the model already has same-lab rows for those (46.13, 69.06, Arena Hard v2 43.1) from its own card. Aggregation not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '34' for 'AllenAI IFBench'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "AllenAI IFBench",
      "column_label": "Mistral Small 3.2",
      "raw_value": "34",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "mistral-small-3-2",
    "bench": "mmmu-pro",
    "score": 0.491,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "instruct",
    "note": "Mistral own results in the pinned repository chart images/image2.png ('Performance comparison across internal models', embedded under README 'Comparison with internal models'; visually read; archived copy hashed in evidence). Same-lab column 'Mistral Small 3.2' (single Instruct bar; Mistral-Small-3.2-24B-Instruct-2506). Its GPQA Diamond 50, MMLU Pro 69 and Arena Hard 43.1 are not added: the model already has same-lab rows for those (46.13, 69.06, Arena Hard v2 43.1) from its own card. Vision benchmark; setting not stated. Mistral Medium 3.1 is not in the catalogue; AA LCR not used. Printed label '49.1' for 'MMMU-Pro'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image2.png",
      "row_label": "MMMU-Pro",
      "column_label": "Mistral Small 3.2",
      "raw_value": "49.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image2.png",
      "asset_sha256": "abde364d45a8f25096dbe279b15db1acd1d4ea6f74fe858281266d5748ffd377",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image2.png"
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "collie",
    "score": 0.603,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "reasoning",
    "note": "Mistral own results in the pinned repository chart images/image3.png ('Performance Comparison Across Internal Models', embedded under README 'Comparing Reasoning Models'; visually read; archived copy hashed in evidence). Same-lab column 'Magistral Small 1.2' (= Magistral-Small-2509). Its AIME25 80.2 is not added (existing aime-2025 row 77.34 from its own card); AA LCR not used. COLLIE; subset and aggregation not stated. Magistral Medium 1.2 is not in the catalogue; AA LCR (custom implementation) not used. Printed label '60.3' for 'Collie'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image3.png",
      "row_label": "Collie",
      "column_label": "Magistral Small 1.2",
      "raw_value": "60.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image3.png",
      "asset_sha256": "b38c8a96b124ff831fa9e7270939123565b396485b48f3219f0387142661b735",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image3.png"
    }
  },
  {
    "model": "magistral-small-2509",
    "bench": "livecodebench-unversioned",
    "score": 0.607,
    "source": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md",
    "source_id": "card-mistralai-mistral-small-4-119b-2603",
    "by": "lab",
    "date": "2026-09-25",
    "mode": "reasoning",
    "note": "Mistral own results in the pinned repository chart images/image3.png ('Performance Comparison Across Internal Models', embedded under README 'Comparing Reasoning Models'; visually read; archived copy hashed in evidence). Same-lab column 'Magistral Small 1.2' (= Magistral-Small-2509). Its AIME25 80.2 is not added (existing aime-2025 row 77.34 from its own card); AA LCR not used. Version/window not stated (the Mistral Small 4 bar in the same chart is recorded as livecodebench-unversioned). Magistral Medium 1.2 is not in the catalogue; AA LCR (custom implementation) not used. Printed label '60.7' for 'LiveCodeBench'. Review date is not the evaluation-run date.",
    "evidence": {
      "kind": "chart-label",
      "quote": "images/image3.png",
      "row_label": "LiveCodeBench",
      "column_label": "Magistral Small 1.2",
      "raw_value": "60.7",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/images/image3.png",
      "asset_sha256": "b38c8a96b124ff831fa9e7270939123565b396485b48f3219f0387142661b735",
      "asset_file": "evidence/mistralai__Mistral-Small-4-119B-2603@a11f36bebf709121056b1dbcc943d1c6afbe494d.images__image3.png"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.563,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. Split and harness not stated. Printed label '56.3' for 'SWE-Bench Pro'; README prose: 'secured a definitive second-place finish on SWE-Bench PRO with a score of 56.3'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "SWE-Bench Pro",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "56.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "secured a definitive second-place finish on SWE-Bench PRO with a score of 56.3"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.595,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. Harness not stated. Printed label '59.5' for 'Terminal-Bench 2.1'; README prose: 'evaluations like Terminal-Bench 2.1 (59.5)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "Terminal-Bench 2.1",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "59.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "evaluations like Terminal-Bench 2.1 (59.5)"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "toolathlon",
    "score": 0.495,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. Printed label '49.5' for 'Toolathlon'; README prose: 'Backed by scores of 49.5 on Toolathlon'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "Toolathlon",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "49.5",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "Backed by scores of 49.5 on Toolathlon"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "simplevqa-with-tools",
    "score": 0.792,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. Search tool allowed. Printed label '79.2' for 'SimpleVQA (with Tool)'; README prose: 'securing first place on SimpleVQA (Search) with a 79.2'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "SimpleVQA (with Tool)",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "79.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "securing first place on SimpleVQA (Search) with a 79.2"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "vstar-with-python",
    "score": 0.953,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. Python tool allowed. Printed label '95.3' for 'V* (with Python)'; README prose: 'achieving frontier parity on V* (Python) at 95.3'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "V* (with Python)",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "95.3",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "achieving frontier parity on V* (Python) at 95.3"
    }
  },
  {
    "model": "step-3-7-flash",
    "bench": "claw-eval-1-1-unspecified",
    "score": 0.671,
    "source": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md",
    "source_id": "card2-stepfun-ai-step-3-7-flash",
    "by": "lab",
    "date": "2026-06-03",
    "mode": "reasoning-level-not-stated",
    "note": "StepFun own results in the benchmark chart committed in the pinned stepfun-ai/Step-3.7-Flash repository (assets/benchmarks.png, embedded in README section 2 'Capabilities & Performance'; visually read; archived copy hashed in evidence). The chart has no legend: the highlighted first bar with the StepFun logo is Step 3.7 Flash, and the README prose states the same value for it. The second (unlabelled) StepFun bar and other labs' bars are not used. Reasoning level (low/medium/high) not stated. HLE (with Tool) is not used because the chart (47.2) and the prose (48.1) disagree; GDPval (GDPVal-AA in the prose) not used. ClawEval-1.1, 2026-05-09 release; Pass^3 vs average not stated. Printed label '67.1' for 'ClawEval-1.1 (2026-05-09)'; README prose: 'Step 3.7 Flash leads the ClawEval-1.1 benchmark with a score of 67.1'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmarks.png",
      "row_label": "ClawEval-1.1 (2026-05-09)",
      "column_label": "Step 3.7 Flash (highlighted first bar, StepFun logo; no legend)",
      "raw_value": "67.1",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/5f6244077ac62e04eec3f320501ff8c2b293373a/assets/benchmarks.png",
      "asset_sha256": "3d26171162c0421a57c6c2c22074b9b276b626c5d90fe3a62e9fceb8ad988ae7",
      "asset_file": "evidence/stepfun-ai__Step-3.7-Flash@5f6244077ac62e04eec3f320501ff8c2b293373a.assets__benchmarks.png",
      "text_quote": "Step 3.7 Flash leads the ClawEval-1.1 benchmark with a score of 67.1"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "tau3-bench",
    "score": 0.729,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Printed value '72.9' for 'τ³-bench'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "τ³-bench",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "72.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "claw-eval-pass3",
    "score": 0.638,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. ClawEval version not stated. Printed value '63.8' for 'ClawEval (pass^3)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "ClawEval (pass^3)",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "63.8",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "hle-with-tools",
    "score": 0.48,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Main cell value; the sub-label 'w.o. tools 34.0' marks it as the with-tools score. Subset (text-only vs full) not stated. Printed value '48.0' for 'Humanity's Last Exam'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "Humanity's Last Exam",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "48.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "hle",
    "score": 0.34,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Sub-label 'w.o. tools 34.0' under the main cell. Subset (text-only vs full) not stated. Printed value '34.0' for 'Humanity's Last Exam (w.o. tools)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "Humanity's Last Exam (w.o. tools)",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "34.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.572,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Split and harness not stated. Printed value '57.2' for 'SWE-bench Pro'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "SWE-bench Pro",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "57.2",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "swe-bench-verified",
    "score": 0.789,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Harness not stated. Printed value '78.9' for 'SWE-bench Verified'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "SWE-bench Verified",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "78.9",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "terminal-bench-2",
    "score": 0.684,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro",
    "by": "lab",
    "date": "2026-07-09",
    "mode": "default (not stated)",
    "note": "Xiaomi MiMo own results in the 'Benchmark Results' table image committed in the pinned XiaomiMiMo/MiMo-V2.5-Pro repository (assets/benchmark.jpg, embedded at the top of the README; visually read; archived copy hashed in evidence). MiMo-V2.5-Pro column only (MiMo-V2-Pro is not in the catalogue; the other columns are other labs). Settings are not stated in the card. GDPVal-AA (Elo) and FrontierSWE (rank) not used. The README's text tables cover MiMo-V2.5-Pro-Base only and are not used; GraphWalks curve points (assets/post_training_evaluation.jpg) not used. Harness not stated. Printed value '68.4' for 'Terminal-Bench 2.0'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "assets/benchmark.jpg",
      "row_label": "Terminal-Bench 2.0",
      "column_label": "MiMo-V2.5-Pro (1.02T / 42B)",
      "raw_value": "68.4",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/21d1ecfecd7bd70f31be25ca49d7edd21f003659/assets/benchmark.jpg",
      "asset_sha256": "5b4e4a2c797c9ab879249f88b5b2341322391f71c689941a70e075b66401a6c6",
      "asset_file": "evidence/XiaomiMiMo__MiMo-V2.5-Pro@21d1ecfecd7bd70f31be25ca49d7edd21f003659.assets__benchmark.jpg"
    }
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "math-500",
    "score": 0.94,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/blob/e9b33add76883f293d6bf61f6bd89b497e80e335/README.md",
    "source_id": "card2-deepseek-ai-deepseek-v3-0324",
    "by": "lab",
    "date": "2025-03-27",
    "mode": "non-thinking",
    "note": "DeepSeek own result in the benchmark chart committed in the pinned DeepSeek-V3-0324 repository (figures/0324_comparison.png, embedded in the README as 'Model Performance'; visually read; archived copy hashed in evidence). Hatched DeepSeek-V3-0324 bar. MATH-500 (Pass@1) is the only chart value not already recorded for this model (MMLU-Pro, GPQA Diamond, AIME 2024 and LiveCodeBench come from the README bullets or the V3.1 card with the same values). DeepSeek-V3 column values are already recorded (its AIME 2024 39.6 differs from the existing 39.2 R1-card row and is not added); Qwen-Max, GPT-4.5 and Claude columns not used. Printed label '94.0' for 'MATH-500 (Pass@1)'.",
    "evidence": {
      "kind": "chart-label",
      "quote": "figures/0324_comparison.png",
      "row_label": "MATH-500 (Pass@1)",
      "column_label": "DeepSeek-V3-0324",
      "raw_value": "94.0",
      "scale_divisor": 100,
      "asset_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/resolve/e9b33add76883f293d6bf61f6bd89b497e80e335/figures/0324_comparison.png",
      "asset_sha256": "7212ae57a3a299c664858e74db102ef70daab173c00a3fa4913deb9d3cddb370",
      "asset_file": "evidence/deepseek-ai__DeepSeek-V3-0324@e9b33add76883f293d6bf61f6bd89b497e80e335.figures__0324_comparison.png"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-tdt-0.6B-v2",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2254,
    "raw_value": 22.54,
    "note": "Table 6 row 'Parakeet-tdt-0.6B-v2': mean WER 22.54% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Nvidia Parakeet-tdt-0.6B-v2'. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 32.6,
      "ghana_wer_percent": 26.27,
      "kenya_wer_percent": 21.78,
      "rwanda_wer_percent": 23.92,
      "uganda_wer_percent": 9.38,
      "south_africa_wer_percent": 21.08,
      "north_africa_wer_percent": 22.76
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-rnnt-1.1B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2563,
    "raw_value": 25.63,
    "note": "Table 6 row 'Parakeet-rnnt-1.1B': mean WER 25.63% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Nvidia Parakeet-rnnt-1.1B'. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 32.76,
      "ghana_wer_percent": 23.69,
      "kenya_wer_percent": 23.19,
      "rwanda_wer_percent": 24.71,
      "uganda_wer_percent": 23.08,
      "south_africa_wer_percent": 27.59,
      "north_africa_wer_percent": 24.42
    }
  },
  {
    "model": "whisper-large-v3",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Whisper-large-v3",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2119,
    "raw_value": 21.19,
    "note": "Table 6 row 'Whisper-large-v3': mean WER 21.19% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'OpenAI Whisper-large-v3'. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 26.53,
      "ghana_wer_percent": 27.22,
      "kenya_wer_percent": 16.7,
      "rwanda_wer_percent": 21.18,
      "uganda_wer_percent": 16.16,
      "south_africa_wer_percent": 19.85,
      "north_africa_wer_percent": 20.72
    }
  },
  {
    "model": "distil-large-v3-5",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Distil-Whisper-v3.5",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2228,
    "raw_value": 22.28,
    "note": "Table 6 row 'Distil-Whisper-v3.5': mean WER 22.28% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: §3.3 footnote https://huggingface.co/distil-whisper/distil-large-v3.5. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 26.69,
      "ghana_wer_percent": 22.16,
      "kenya_wer_percent": 18.51,
      "rwanda_wer_percent": 23.53,
      "uganda_wer_percent": 19.22,
      "south_africa_wer_percent": 24.45,
      "north_africa_wer_percent": 21.43
    }
  },
  {
    "model": "voxtral-mini-3b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Voxtral (Mistral)",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2287,
    "raw_value": 22.87,
    "note": "Table 6 row 'Voxtral (Mistral)': mean WER 22.87% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Mistral Voxtral-Mini-3B' (open weights; only the 2507 release existed at submission, Nov 2025). Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 27.84,
      "ghana_wer_percent": 22.49,
      "kenya_wer_percent": 21.5,
      "rwanda_wer_percent": 23.53,
      "uganda_wer_percent": 17.57,
      "south_africa_wer_percent": 26.96,
      "north_africa_wer_percent": 20.17
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Canary-Qwen-2.5B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2237,
    "raw_value": 22.37,
    "note": "Table 6 row 'Canary-Qwen-2.5B': mean WER 22.37% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: §3.3 footnote https://huggingface.co/nvidia/canary-qwen-2.5b. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 26.82,
      "ghana_wer_percent": 22.1,
      "kenya_wer_percent": 18.49,
      "rwanda_wer_percent": 21.57,
      "uganda_wer_percent": 21.45,
      "south_africa_wer_percent": 25.19,
      "north_africa_wer_percent": 20.99
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "GPT-4o Transcribe",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.3894,
    "raw_value": 38.94,
    "note": "Table 6 row 'GPT-4o Transcribe': mean WER 38.94% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: §3.3 footnote https://platform.openai.com/docs/models/gpt-4o-transcribe. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 40.03,
      "ghana_wer_percent": 60.41,
      "kenya_wer_percent": 58.38,
      "rwanda_wer_percent": 15.69,
      "uganda_wer_percent": 15.22,
      "south_africa_wer_percent": 60.43,
      "north_africa_wer_percent": 22.4
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-tdt-0.6B-v2",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.1845,
    "raw_value": 18.45,
    "note": "Table 3 row 'Parakeet-tdt-0.6B-v2', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 18.45%. Identity: Table 2 'Nvidia Parakeet-tdt-0.6B-v2'. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 30.2,
      "dialog_wer_percent": 11.23,
      "med_conv_nig_wer_percent": 29.41,
      "afrinames_wer_percent": 41.88,
      "call_center_wer_percent": 20.96,
      "robustness_wer_percent": 40.89
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-rnnt-1.1B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.2675,
    "raw_value": 26.75,
    "note": "Table 3 row 'Parakeet-rnnt-1.1B', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 26.75%. Identity: Table 2 'Nvidia Parakeet-rnnt-1.1B'. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 28.18,
      "dialog_wer_percent": 15.08,
      "med_conv_nig_wer_percent": 30.59,
      "afrinames_wer_percent": 46.7,
      "call_center_wer_percent": 28.93,
      "robustness_wer_percent": 90.3
    }
  },
  {
    "model": "whisper-large-v3",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Whisper-large-v3",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.1999,
    "raw_value": 19.99,
    "note": "Table 3 row 'Whisper-large-v3', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 19.99%. Identity: Table 2 'OpenAI Whisper-large-v3'. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 26.49,
      "dialog_wer_percent": 13.49,
      "med_conv_nig_wer_percent": 31.76,
      "afrinames_wer_percent": 43.23,
      "call_center_wer_percent": 24.69,
      "robustness_wer_percent": 33.79
    }
  },
  {
    "model": "distil-large-v3-5",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Distil-Whisper-v3.5",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.18,
    "raw_value": 18,
    "note": "Table 3 row 'Distil-Whisper-v3.5', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 18.0%. Identity: §3.3 footnote https://huggingface.co/distil-whisper/distil-large-v3.5. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 27.58,
      "dialog_wer_percent": 11.5,
      "med_conv_nig_wer_percent": 30.41,
      "afrinames_wer_percent": 45.8,
      "call_center_wer_percent": 21.65,
      "robustness_wer_percent": 34
    }
  },
  {
    "model": "voxtral-mini-3b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Voxtral (Mistral)",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.211,
    "raw_value": 21.1,
    "note": "Table 3 row 'Voxtral (Mistral)', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 21.1%. Identity: Table 2 'Mistral Voxtral-Mini-3B' (open weights; only the 2507 release existed at submission, Nov 2025). The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 20.17,
      "dialog_wer_percent": 68.42,
      "med_conv_nig_wer_percent": 78.73,
      "afrinames_wer_percent": 49.36,
      "call_center_wer_percent": 29.2,
      "robustness_wer_percent": 43.51
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Canary-Qwen-2.5B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.1818,
    "raw_value": 18.18,
    "note": "Table 3 row 'Canary-Qwen-2.5B', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 18.18%. Identity: §3.3 footnote https://huggingface.co/nvidia/canary-qwen-2.5b. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 29.87,
      "dialog_wer_percent": 96.64,
      "med_conv_nig_wer_percent": 97.89,
      "afrinames_wer_percent": 42.91,
      "call_center_wer_percent": 39.09,
      "robustness_wer_percent": 41.15
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "GPT-4o Transcribe",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.6439,
    "raw_value": 64.39,
    "note": "Table 3 row 'GPT-4o Transcribe', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 64.39%. Identity: §3.3 footnote https://platform.openai.com/docs/models/gpt-4o-transcribe. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 24.66,
      "dialog_wer_percent": 15.03,
      "med_conv_nig_wer_percent": 30.8,
      "afrinames_wer_percent": 52.49,
      "call_center_wer_percent": 23.2,
      "robustness_wer_percent": 33.59
    }
  },
  {
    "model": "google-chirp-3",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Google Chirp V3",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.2818,
    "raw_value": 28.18,
    "note": "Table 3 row 'Google Chirp V3', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 28.18%. Identity: §3.3 footnote https://cloud.google.com/speech-to-text/v2/docs/chirp_3-model (Chirp 3). The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 35.03,
      "dialog_wer_percent": 17.53,
      "med_conv_nig_wer_percent": 38.57,
      "afrinames_wer_percent": 52.45,
      "call_center_wer_percent": 29.6,
      "robustness_wer_percent": 31.7
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3686,
    "raw_value": 36.86,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-large-v3': Average WER 36.86% over six Arabic test sets (sada 55.96, commonvoice18 17.83, masc_clean 24.66, masc_noisy 34.63, mgb2 16.26, casablanca 71.81); Average CER 17.21%. HF id openai/whisper-large-v3. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-large-v3",
    "components": {
      "sada": 55.96,
      "commonvoice18": 17.83,
      "masc_clean": 24.66,
      "masc_noisy": 34.63,
      "mgb2": 16.26,
      "casablanca": 71.81
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4005,
    "raw_value": 40.05,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-large-v3-turbo': Average WER 40.05% over six Arabic test sets (sada 60.36, commonvoice18 25.73, masc_clean 25.51, masc_noisy 37.16, mgb2 17.75, casablanca 73.79); Average CER 18.87%. HF id openai/whisper-large-v3-turbo. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-large-v3-turbo",
    "components": {
      "sada": 60.36,
      "commonvoice18": 25.73,
      "masc_clean": 25.51,
      "masc_noisy": 37.16,
      "mgb2": 17.75,
      "casablanca": 73.79
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3447,
    "raw_value": 34.47,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-09-22",
    "date_basis": "Space commit 2ef7c9acec first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'mistralai/Voxtral-Small-24B-2507': Average WER 34.47% over six Arabic test sets (sada 50.82, commonvoice18 15.25, masc_clean 23.96, masc_noisy 34.43, mgb2 16.03, casablanca 66.3); Average CER 15.29%. HF id mistralai/Voxtral-Small-24B-2507. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "mistralai/Voxtral-Small-24B-2507",
    "components": {
      "sada": 50.82,
      "commonvoice18": 15.25,
      "masc_clean": 23.96,
      "masc_noisy": 34.43,
      "mgb2": 16.03,
      "casablanca": 66.3
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4258,
    "raw_value": 42.58,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-09-22",
    "date_basis": "Space commit 2ef7c9acec first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'mistralai/Voxtral-Mini-3B-2507': Average WER 42.58% over six Arabic test sets (sada 63.65, commonvoice18 22.12, masc_clean 28.37, masc_noisy 41.27, mgb2 22.56, casablanca 77.52); Average CER 19.9%. HF id mistralai/Voxtral-Mini-3B-2507. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "mistralai/Voxtral-Mini-3B-2507",
    "components": {
      "sada": 63.65,
      "commonvoice18": 22.12,
      "masc_clean": 28.37,
      "masc_noisy": 41.27,
      "mgb2": 22.56,
      "casablanca": 77.52
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3336,
    "raw_value": 33.36,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2026-01-30",
    "date_basis": "Space commit 8eab4f017f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'Qwen/Qwen3-ASR-1.7B': Average WER 33.36% over six Arabic test sets (sada 45.53, commonvoice18 16.9, masc_clean 24.37, masc_noisy 34.29, mgb2 16.57, casablanca 64.47); Average CER 12.33%. HF id Qwen/Qwen3-ASR-1.7B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. FLAG: the six per-set WERs average 33.688%, not the published 33.36% (source inconsistency; published Average kept).",
    "engine_label": "Qwen/Qwen3-ASR-1.7B",
    "components": {
      "sada": 45.53,
      "commonvoice18": 16.9,
      "masc_clean": 24.37,
      "masc_noisy": 34.29,
      "mgb2": 16.57,
      "casablanca": 64.47
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4219,
    "raw_value": 42.19,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2026-01-30",
    "date_basis": "Space commit a2f2a99442 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'Qwen/Qwen3-ASR-0.6B': Average WER 42.19% over six Arabic test sets (sada 53.75, commonvoice18 28.28, masc_clean 31.34, masc_noisy 42.63, mgb2 25.45, casablanca 71.68); Average CER 16.23%. HF id Qwen/Qwen3-ASR-0.6B (proposed comparison-only model; also on the Persian Triple Threat board). Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "Qwen/Qwen3-ASR-0.6B",
    "components": {
      "sada": 53.75,
      "commonvoice18": 28.28,
      "masc_clean": 31.34,
      "masc_noisy": 42.63,
      "mgb2": 25.45,
      "casablanca": 71.68
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3067,
    "raw_value": 30.67,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2026-04-16",
    "date_basis": "Space commit df78df8603 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'CohereLabs/cohere-transcribe-03-2026': Average WER 30.67% over six Arabic test sets (sada 60.11, commonvoice18 8.17, masc_clean 8.66, masc_noisy 19.01, mgb2 25.33, casablanca 62.71); Average CER 16.37%. HF id CohereLabs/cohere-transcribe-03-2026 (the general model, not the separate cohere-transcribe-arabic-07-2026 row, which is not mapped). Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "CohereLabs/cohere-transcribe-03-2026",
    "components": {
      "sada": 60.11,
      "commonvoice18": 8.17,
      "masc_clean": 8.66,
      "masc_noisy": 19.01,
      "mgb2": 25.33,
      "casablanca": 62.71
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v3",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.087543,
    "raw_value": 8.7543,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v3' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.08754281957017103 (fraction as published) on 531 recordings. Whisper large-v3 (openai/whisper-large-v3); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper large-v3",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.307481,
    "raw_value": 30.7481,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v3' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.3074807889458775 (fraction as published) on 3257 recordings. Whisper large-v3 (openai/whisper-large-v3); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper large-v3",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.101452,
    "raw_value": 10.1452,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v3-turbo' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.10145202886960648 (fraction as published) on 531 recordings. Whisper large-v3-turbo (openai/whisper-large-v3-turbo); reproduction agrees with the released scores to within 0.04 pp (reproducibility-audit.csv).",
    "engine_label": "Whisper large-v3-turbo",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.3313,
    "raw_value": 33.13,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v3-turbo' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.331299665375507 (fraction as published) on 3257 recordings. Whisper large-v3-turbo (openai/whisper-large-v3-turbo); reproduction agrees with the released scores to within 0.04 pp (reproducibility-audit.csv).",
    "engine_label": "Whisper large-v3-turbo",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.12111,
    "raw_value": 12.111,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Voxtral Mini 3B 2507' (run_type additional_benchmark, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.12111023673901836 (fraction as published) on 531 recordings. Label 'Voxtral Mini 3B 2507' = mistralai/Voxtral-Mini-3B-2507; additional full-split benchmark run by the board maintainer.",
    "engine_label": "Voxtral Mini 3B 2507",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.339806,
    "raw_value": 33.9806,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Voxtral Mini 3B 2507' (run_type additional_benchmark, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.33980595958731025 (fraction as published) on 3257 recordings. Label 'Voxtral Mini 3B 2507' = mistralai/Voxtral-Mini-3B-2507; additional full-split benchmark run by the board maintainer.",
    "engine_label": "Voxtral Mini 3B 2507",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.155562,
    "raw_value": 15.5562,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Parakeet TDT 0.6B v3' (run_type additional_benchmark, status validated, metric_basis mean, 5 blank outputs): standard_audio_wer 0.15556194873415696 (fraction as published) on 531 recordings. Label 'Parakeet TDT 0.6B v3' = nvidia/parakeet-tdt-0.6b-v3; additional full-split benchmark run by the board maintainer (5 blank outputs).",
    "engine_label": "Parakeet TDT 0.6B v3",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.430908,
    "raw_value": 43.0908,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Parakeet TDT 0.6B v3' (run_type additional_benchmark, status validated, metric_basis mean, 5 blank outputs): std_reference_wer 0.43090841134275526 (fraction as published) on 3257 recordings. Label 'Parakeet TDT 0.6B v3' = nvidia/parakeet-tdt-0.6b-v3; additional full-split benchmark run by the board maintainer (5 blank outputs).",
    "engine_label": "Parakeet TDT 0.6B v3",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3",
    "bench": "botnoi-thai-telephony-cer",
    "score": 0.4844,
    "raw_value": 48.44,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "Whisper-large-v3",
    "data_license": "CC BY 4.0",
    "note": "Table 3 row 'Whisper-large-v3': CER 48.44% on the BOTNOI Thai telephony evaluation set (in-house real-call and prompted telephony recordings; zero-shot baseline, BOTNOI's own run).",
    "components": {}
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "botnoi-thai-telephony-cer",
    "score": 0.1968,
    "raw_value": 19.68,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "ElevenLabs Scribe v2",
    "data_license": "CC BY 4.0",
    "note": "Table 3 row 'ElevenLabs Scribe v2': CER 19.68% on the BOTNOI Thai telephony evaluation set (in-house real-call and prompted telephony recordings; zero-shot baseline, BOTNOI's own run).",
    "components": {}
  },
  {
    "model": "google-chirp-3",
    "bench": "botnoi-thai-telephony-cer",
    "score": 0.1342,
    "raw_value": 13.42,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "Google Chirp 3",
    "data_license": "CC BY 4.0",
    "note": "Table 3 row 'Google Chirp 3': CER 13.42% on the BOTNOI Thai telephony evaluation set (in-house real-call and prompted telephony recordings; zero-shot baseline, BOTNOI's own run).",
    "components": {}
  },
  {
    "model": "whisper-large-v3",
    "bench": "botnoi-thai-fleurs-cer",
    "score": 0.0885,
    "raw_value": 8.85,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "Whisper-large-v3",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Whisper-large-v3': CER 8.85% on FLEURS Thai (Common Voice 23 Thai CER 7.25% kept in components). Zero-shot baseline run by BOTNOI.",
    "components": {
      "common_voice_23_th_cer_percent": 7.25
    }
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "botnoi-thai-fleurs-cer",
    "score": 0.0623,
    "raw_value": 6.23,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "ElevenLabs Scribe v2",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'ElevenLabs Scribe v2': CER 6.23% on FLEURS Thai (Common Voice 23 Thai CER 1.92% kept in components). Zero-shot baseline run by BOTNOI.",
    "components": {
      "common_voice_23_th_cer_percent": 1.92
    }
  },
  {
    "model": "google-chirp-3",
    "bench": "botnoi-thai-fleurs-cer",
    "score": 0.1029,
    "raw_value": 10.29,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2608.24916v1",
    "source_id": "speech-stt-botnoi-thai-report",
    "by": "independent",
    "date": "2026-08-03",
    "date_basis": "arXiv v1 submission date (3 Aug 2026); run date not stated",
    "engine_label": "Google Chirp 3",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Google Chirp 3': CER 10.29% on FLEURS Thai (Common Voice 23 Thai CER 3.46% kept in components). Zero-shot baseline run by BOTNOI.",
    "components": {
      "common_voice_23_th_cer_percent": 3.46
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.0826,
    "raw_value": 8.26,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id Transformers_2025-10-27_00-23-30 in results.csv",
    "note": "model_id 'openai/whisper-large-v3' (exact HF id): Average WER 8.26% = mean of google_chilean_spanish 4.6% (4,374 utt), common_voice 3.64% (152), datarisas 16.53% (50), recomputed as the Space's load_results() does (round 2). Job Transformers_2025-10-27_00-23-30.",
    "engine_label": "openai/whisper-large-v3",
    "sample_count": 4576,
    "components": {
      "datarisas": 16.53,
      "google_chilean_spanish": 4.6,
      "common_voice": 3.64
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.0829,
    "raw_value": 8.29,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-26",
    "date_basis": "run job id Transformers_2025-10-26_23-22-40 in results.csv",
    "note": "model_id 'openai/whisper-large-v3-turbo' (exact HF id): Average WER 8.29% = mean of google_chilean_spanish 2.86% (4,374 utt), common_voice 4.94% (152), datarisas 17.07% (50), recomputed as the Space's load_results() does (round 2). Job Transformers_2025-10-26_23-22-40.",
    "engine_label": "openai/whisper-large-v3-turbo",
    "sample_count": 4576,
    "components": {
      "google_chilean_spanish": 2.86,
      "datarisas": 17.07,
      "common_voice": 4.94
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.079,
    "raw_value": 7.9,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id NeMo_2025-10-27_00-26-07 in results.csv",
    "note": "model_id 'nvidia/parakeet-tdt-0.6b-v3' (exact HF id): Average WER 7.9% = mean of google_chilean_spanish 4.44% (4,374 utt), common_voice 2.86% (152), datarisas 16.4% (50), recomputed as the Space's load_results() does (round 2). Job NeMo_2025-10-27_00-26-07.",
    "engine_label": "nvidia/parakeet-tdt-0.6b-v3",
    "sample_count": 4576,
    "components": {
      "datarisas": 16.4,
      "google_chilean_spanish": 4.44,
      "common_voice": 2.86
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "canary-1b-v2",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.0982,
    "raw_value": 9.82,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id NeMo_2025-10-27_00-27-29 in results.csv",
    "note": "model_id 'nvidia/canary-1b-v2' (exact HF id): Average WER 9.82% = mean of google_chilean_spanish 4.95% (4,374 utt), common_voice 3.58% (152), datarisas 20.93% (50), recomputed as the Space's load_results() does (round 2). Job NeMo_2025-10-27_00-27-29.",
    "engine_label": "nvidia/canary-1b-v2",
    "sample_count": 4576,
    "components": {
      "google_chilean_spanish": 4.95,
      "datarisas": 20.93,
      "common_voice": 3.58
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.0834,
    "raw_value": 8.34,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id Voxtral_2025-10-27_00-29-28 in results.csv",
    "note": "model_id 'mistralai/Voxtral-Mini-3B-2507' (exact HF id): Average WER 8.34% = mean of google_chilean_spanish 4.65% (4,374 utt), common_voice 3.58% (152), datarisas 16.8% (50), recomputed as the Space's load_results() does (round 2). Job Voxtral_2025-10-27_00-29-28.",
    "engine_label": "mistralai/Voxtral-Mini-3B-2507",
    "sample_count": 4576,
    "components": {
      "google_chilean_spanish": 4.65,
      "datarisas": 16.8,
      "common_voice": 3.58
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.1735,
    "raw_value": 17.35,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "GPT-4o-trans.",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'GPT-4o-trans.': overall Avg. WER 17.35% = unweighted mean of the CHiME-4 (6.47), VOiCES (22.65) and NOIZEUS (22.94) averages. Identity: table label 'GPT-4o-trans.' (closed-source block; the baselines paragraph cites the GPT-4o system card). JUDGEMENT CALL: mapped to gpt-4o-transcribe from the abbreviated label; no snapshot date given. Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 5.36,
      "chime4_sim_wer_percent": 7.57,
      "chime4_avg_wer_percent": 6.47,
      "voices_rm1_wer_percent": 10.97,
      "voices_rm2_wer_percent": 12.56,
      "voices_rm3_wer_percent": 46.68,
      "voices_rm4_wer_percent": 29.38,
      "voices_avg_wer_percent": 22.65,
      "noizeus_0db_wer_percent": 62.4,
      "noizeus_5db_wer_percent": 20.56,
      "noizeus_10db_wer_percent": 6.15,
      "noizeus_15db_wer_percent": 2.64,
      "noizeus_avg_wer_percent": 22.94
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.1215,
    "raw_value": 12.15,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "Voxtral-Mini",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Voxtral-Mini': overall Avg. WER 12.15% = unweighted mean of the CHiME-4 (7.53), VOiCES (12.75) and NOIZEUS (16.16) averages. Identity: baselines list 'Voxtral-Mini-3B' (Liu et al., 2025); Table 3 labels it Voxtral-Mini-3B-2507. Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 6.01,
      "chime4_sim_wer_percent": 9.04,
      "chime4_avg_wer_percent": 7.53,
      "voices_rm1_wer_percent": 3.5,
      "voices_rm2_wer_percent": 3.51,
      "voices_rm3_wer_percent": 27.54,
      "voices_rm4_wer_percent": 16.45,
      "voices_avg_wer_percent": 12.75,
      "noizeus_0db_wer_percent": 41.06,
      "noizeus_5db_wer_percent": 15.8,
      "noizeus_10db_wer_percent": 4.85,
      "noizeus_15db_wer_percent": 2.94,
      "noizeus_avg_wer_percent": 16.16
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.1072,
    "raw_value": 10.72,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "Whisper-L-v3",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Whisper-L-v3': overall Avg. WER 10.72% = unweighted mean of the CHiME-4 (7.02), VOiCES (11.79) and NOIZEUS (13.34) averages. Identity: baselines list 'Whisper-Large-v3' (Radford et al.). Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 5.65,
      "chime4_sim_wer_percent": 8.39,
      "chime4_avg_wer_percent": 7.02,
      "voices_rm1_wer_percent": 2.85,
      "voices_rm2_wer_percent": 2.97,
      "voices_rm3_wer_percent": 25.68,
      "voices_rm4_wer_percent": 15.65,
      "voices_avg_wer_percent": 11.79,
      "noizeus_0db_wer_percent": 34.71,
      "noizeus_5db_wer_percent": 12.55,
      "noizeus_10db_wer_percent": 3.93,
      "noizeus_15db_wer_percent": 2.17,
      "noizeus_avg_wer_percent": 13.34
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.1184,
    "raw_value": 11.84,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "Canary-1B-v2",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Canary-1B-v2': overall Avg. WER 11.84% = unweighted mean of the CHiME-4 (8.46), VOiCES (11.65) and NOIZEUS (15.41) averages. Identity: baselines list 'Canary-1B-v2' (Sekoyan et al., 2025). Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 7.19,
      "chime4_sim_wer_percent": 9.73,
      "chime4_avg_wer_percent": 8.46,
      "voices_rm1_wer_percent": 3.14,
      "voices_rm2_wer_percent": 3,
      "voices_rm3_wer_percent": 24.88,
      "voices_rm4_wer_percent": 15.56,
      "voices_avg_wer_percent": 11.65,
      "noizeus_0db_wer_percent": 38.53,
      "noizeus_5db_wer_percent": 12.76,
      "noizeus_10db_wer_percent": 6.56,
      "noizeus_15db_wer_percent": 3.77,
      "noizeus_avg_wer_percent": 15.41
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.1115,
    "raw_value": 11.15,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "Parakeet-v3",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Parakeet-v3': overall Avg. WER 11.15% = unweighted mean of the CHiME-4 (7.72), VOiCES (10.03) and NOIZEUS (15.69) averages. Identity: baselines list 'Parakeet-TDT-0.6B-v3' (Sekoyan et al., 2025); Table 3 labels it Parakeet-TDT-0.6B-v3. Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 6.61,
      "chime4_sim_wer_percent": 8.82,
      "chime4_avg_wer_percent": 7.72,
      "voices_rm1_wer_percent": 3.23,
      "voices_rm2_wer_percent": 3.27,
      "voices_rm3_wer_percent": 19.77,
      "voices_rm4_wer_percent": 13.84,
      "voices_avg_wer_percent": 10.03,
      "noizeus_0db_wer_percent": 38.95,
      "noizeus_5db_wer_percent": 14.67,
      "noizeus_10db_wer_percent": 5.99,
      "noizeus_15db_wer_percent": 3.15,
      "noizeus_avg_wer_percent": 15.69
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "megaasr-adverse3-mean-wer",
    "score": 0.0793,
    "raw_value": 7.93,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.19833v1",
    "source_id": "speech-stt-megaasr-report",
    "by": "independent",
    "date": "2026-05-19",
    "date_basis": "arXiv v1 submission date (19 May 2026); run date not stated",
    "engine_label": "Qwen3-ASR",
    "data_license": "CC BY 4.0",
    "note": "Table 2 row 'Qwen3-ASR': overall Avg. WER 7.93% = unweighted mean of the CHiME-4 (5.39), VOiCES (8.94) and NOIZEUS (9.45) averages. Identity: Table 10 repeats this row's CHiME-4/NOIZEUS/VOiCES/Avg values (5.39/9.45/8.94/7.93) as 'Qwen3-ASR-1.7B', the checkpoint Mega-ASR is initialised from. Baseline run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab; not the model makers); per-subset values in components.",
    "components": {
      "chime4_real_wer_percent": 4.66,
      "chime4_sim_wer_percent": 6.11,
      "chime4_avg_wer_percent": 5.39,
      "voices_rm1_wer_percent": 2.52,
      "voices_rm2_wer_percent": 2.62,
      "voices_rm3_wer_percent": 19.18,
      "voices_rm4_wer_percent": 11.44,
      "voices_avg_wer_percent": 8.94,
      "noizeus_0db_wer_percent": 23.97,
      "noizeus_5db_wer_percent": 8.47,
      "noizeus_10db_wer_percent": 3.41,
      "noizeus_15db_wer_percent": 1.96,
      "noizeus_avg_wer_percent": 9.45
    }
  },
  {
    "model": "deepgram-nova-3",
    "bench": "openbench-earnings22-kws-nokw-wer",
    "score": 0.1534,
    "raw_value": 15.34,
    "scale_divisor": 100,
    "source": "https://github.com/argmaxinc/OpenBench/blob/e13108070b9693f2646e4a3e22e4b2d0130d779f/BENCHMARKS.md",
    "source_id": "speech-stt-openbench-mit",
    "by": "independent",
    "date": "2025-12-12",
    "date_basis": "BENCHMARKS.md 'Latest Run: 2025-12-12'",
    "note": "Row 'Deepgram (nova-3)': earnings22-keywords (no keywords) WER 15.34%. Model Version 'nova-3'; Deepgram Python SDK file transcription (DeepgramTranscriptionPipeline.yaml model_version: nova-3). Cloud API, pre-recorded. Keyword-boosted columns (chunk/file keywords) not used.",
    "engine_label": "Deepgram (nova-3)",
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "openbench-earnings22-kws-nokw-wer",
    "score": 0.154,
    "raw_value": 15.4,
    "scale_divisor": 100,
    "source": "https://github.com/argmaxinc/OpenBench/blob/e13108070b9693f2646e4a3e22e4b2d0130d779f/BENCHMARKS.md",
    "source_id": "speech-stt-openbench-mit",
    "by": "independent",
    "date": "2025-12-12",
    "date_basis": "BENCHMARKS.md 'Latest Run: 2025-12-12'",
    "note": "Row 'Whisper OSS (large-v3-turbo)': earnings22-keywords (no keywords) WER 15.4%. Model Version 'large-v3-turbo'; open-source openai-whisper implementation on M2 Ultra Mac Studio. Keyword-boosted columns (chunk/file keywords) not used.",
    "engine_label": "Whisper OSS (large-v3-turbo)",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "openbench-earnings22-kws-nokw-wer",
    "score": 0.1053,
    "raw_value": 10.53,
    "scale_divisor": 100,
    "source": "https://github.com/argmaxinc/OpenBench/blob/e13108070b9693f2646e4a3e22e4b2d0130d779f/BENCHMARKS.md",
    "source_id": "speech-stt-openbench-mit",
    "by": "independent",
    "date": "2026-01-14",
    "date_basis": "row added in commit 955e5786 (2026-01-14), no run date printed",
    "note": "Row 'ElevenLabs': earnings22-keywords (no keywords) WER 10.53%. Row label 'ElevenLabs' has no printed model version; the same commit (955e5786, 2026-01-14) added transcription_elevenlabs.py with default model_id='scribe_v2' and ElevenLabsTranscriptionPipeline.yaml 'model_id: \"scribe_v2\"', unchanged to the pinned commit. ElevenLabs cloud API, pre-recorded. Keyword-boosted columns (chunk/file keywords) not used.",
    "engine_label": "ElevenLabs",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "persian-triple-threat-2split-wer",
    "score": 0.04135,
    "raw_value": 4.135,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Reza2kn/PersianASR-TrippleThreat/blob/63e7e3f8b3754a92d6c0ac2a9db84344cf7e49bb/results.json",
    "source_id": "speech-stt-persian-triple-threat-apache",
    "by": "independent",
    "date": "2026-08-25",
    "date_basis": "Space commit a3fdcca083 (\"Add complete ElevenLabs Scribe v2 benchmark score\") published this row; values unchanged to pinned revision 63e7e3f8b3",
    "note": "Row 'ElevenLabs Scribe v2': VisualEars6669 fair WER 4.22% (n=6669), FLEURS-fa WER 4.05% (n=4341); equal-weight mean 4.135%. Direct ElevenLabs SDK run with Scribe v2 (language_code=fas, tag_audio_events=false, diarize=false), per the row notes.",
    "engine_label": "ElevenLabs Scribe v2",
    "sample_count": 11010,
    "components": {
      "visualears6669_fair_wer": 4.22,
      "fleurs_fa_wer": 4.05
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "gemini-3-5-transcribe",
    "bench": "persian-triple-threat-2split-wer",
    "score": 0.064073,
    "raw_value": 6.4073,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Reza2kn/PersianASR-TrippleThreat/blob/63e7e3f8b3754a92d6c0ac2a9db84344cf7e49bb/gemini35_transcribe_scores.json",
    "source_id": "speech-stt-persian-triple-threat-apache",
    "by": "independent",
    "date": "2026-08-28",
    "date_basis": "Space commit c0a4062052 (\"Add Gemini 3.5 Transcribe Triple Threat benchmark\") published this row; values unchanged to pinned revision 63e7e3f8b3",
    "note": "Row 'Gemini 3.5 Transcribe': VisualEars6669 fair WER 7.2599% (n=6669), FLEURS-fa WER 5.5547% (n=4341); equal-weight mean 6.4073% (identical to the receipt's macro_equal_split WER). Direct Google Gemini API (official google-genai SDK, Files API) with model 'gemini-3.5-transcribe', fa-IR language hint, verbatim mode; receipt gemini35_transcribe_scores.json records model 'gemini-3.5-transcribe', 11010/11010 predictions, 0 missing.",
    "engine_label": "Gemini 3.5 Transcribe",
    "sample_count": 11010,
    "components": {
      "visualears6669_fair_wer": 7.259906775296774,
      "fleurs_fa_wer": 5.554707912955582
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "persian-triple-threat-2split-wer",
    "score": 0.52205,
    "raw_value": 52.205,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Reza2kn/PersianASR-TrippleThreat/blob/63e7e3f8b3754a92d6c0ac2a9db84344cf7e49bb/results.json",
    "source_id": "speech-stt-persian-triple-threat-apache",
    "by": "independent",
    "date": "2026-07-06",
    "date_basis": "Space commit 434043b4fe (compound-fair re-score upload; value unchanged since) published this row; values unchanged to pinned revision 63e7e3f8b3",
    "note": "Row 'Qwen/Qwen3-ASR-0.6B': VisualEars6669 fair WER 58.65% (n=6669), FLEURS-fa WER 45.76% (n=4341); equal-weight mean 52.205%. Open weights Qwen/Qwen3-ASR-0.6B run locally (bf16).",
    "engine_label": "Qwen/Qwen3-ASR-0.6B",
    "sample_count": 11010,
    "components": {
      "visualears6669_fair_wer": 58.65,
      "fleurs_fa_wer": 45.76
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v3",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.059698,
    "raw_value": 5.9698,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T130805Z (run.json status success)",
    "note": "model_id 'whisper-large-v3' = HF openai/whisper-large-v3 @ 06f233fe06e710322aca913c1bc4249a0d71fce1 (transformers-whisper, float16, language fa). Corpus CER 5.97% (95% CI 5.58-6.40); WER 20.37% (95% CI 19.57-21.19); 871/871 recordings; run fleurs-fa-ir-v1-whisper-large-v3-20260817T130805Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.055764760944349206,
      0.06398144080093214
    ],
    "sample_count": 871,
    "engine_label": "whisper-large-v3",
    "data_license": "Apache-2.0",
    "published_value": 0.05969803405044149,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.06104,
    "raw_value": 6.104,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T133753Z (run.json status success)",
    "note": "model_id 'whisper-large-v3-turbo' = HF openai/whisper-large-v3-turbo @ 41f01f3fe87f28c78e2fbf8b568835947dd65ed9 (transformers-whisper, float16, language fa). Corpus CER 6.10% (95% CI 5.64-6.63); WER 20.27% (95% CI 19.45-21.11); 871/871 recordings; run fleurs-fa-ir-v1-whisper-large-v3-turbo-20260817T133753Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.056352841459862954,
      0.06629199329723903
    ],
    "sample_count": 871,
    "engine_label": "whisper-large-v3-turbo",
    "data_license": "Apache-2.0",
    "published_value": 0.061039562905507584,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.089248,
    "raw_value": 8.9248,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-12",
    "date_basis": "run_id timestamp 20260812T010849Z (run.json status success)",
    "note": "model_id 'qwen3-asr-1-7b' = HF Qwen/Qwen3-ASR-1.7B-hf @ bcd2b5b7f32b480ab5790554cfa8347f246a14f3 (transformers-qwen, bfloat16, language fa). Corpus CER 8.92% (95% CI 8.45-9.44); WER 24.10% (95% CI 23.24-25.01); 871/871 recordings; run fleurs-fa-ir-v1-qwen3-asr-1-7b-20260812T010849Z, image ghcr.io/armanjr/peste-benchmark:2.0.0. Qwen/Qwen3-ASR-1.7B-hf is the transformers-format release of Qwen3-ASR-1.7B.",
    "ci95": [
      0.08445951963813145,
      0.09437146140582989
    ],
    "sample_count": 871,
    "engine_label": "qwen3-asr-1-7b",
    "data_license": "Apache-2.0",
    "published_value": 0.08924825601248841,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-large-v3",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.101,
    "raw_value": 10.1,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/whisper-large-v3.json",
    "note": "Row 'whisper-large-v3' (offline). hf_repo openai/whisper-large-v3, transformers ASR pipeline, language=arabic, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 10.1% (CER 4.26%, alef-insensitive WER 8.61%). Subsets: EveryAyah 5.48%, QUL Al-Nufais 4.69%, phone tlog 26.23%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 5.48,
      "qul_alnufais": 4.69,
      "tlog_holdout": 26.23
    },
    "sample_count": 600,
    "engine_label": "whisper-large-v3",
    "data_license": "Apache-2.0"
  },
  {
    "model": "google-chirp-3",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1032,
    "raw_value": 10.32,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/google-chirp3-sync.json",
    "note": "Row 'Google Chirp 3' (offline). Google Speech-to-Text v2 sync recognize with the Chirp 3 model (hf_repo field links the chirp_3 model page), ar-XA, location us; the separate 'Google Chirp 3 (realtime)' streaming row is not used. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 10.32% (CER 6.32%, alef-insensitive WER 8.97%). Subsets: EveryAyah 7.84%, QUL Al-Nufais 10.08%, phone tlog 14.73%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 7.84,
      "qul_alnufais": 10.08,
      "tlog_holdout": 14.73
    },
    "sample_count": 600,
    "engine_label": "Google Chirp 3",
    "data_license": "Apache-2.0"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1247,
    "raw_value": 12.47,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/elevenlabs-scribe-v2.json",
    "note": "Row 'ElevenLabs Scribe v2' (offline). ElevenLabs speech-to-text file API, row labelled 'Scribe v2' (key elevenlabs-scribe-v2), language ar. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 12.47% (CER 5.67%, alef-insensitive WER 11.2%). Subsets: EveryAyah 8.0%, QUL Al-Nufais 13.0%, phone tlog 18.87%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 8,
      "qul_alnufais": 13,
      "tlog_holdout": 18.87
    },
    "sample_count": 600,
    "engine_label": "ElevenLabs Scribe v2",
    "data_license": "Apache-2.0"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1522,
    "raw_value": 15.22,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/cohere-transcribe-03-2026.json",
    "note": "Row 'cohere-transcribe-03-2026' (offline). hf_repo CohereLabs/cohere-transcribe-03-2026, re-run from scratch, language ar (the separate cohere-transcribe-arabic-07-2026 row is a different model and is not used). Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 15.22% (CER 10.42%, alef-insensitive WER 13.82%). Subsets: EveryAyah 13.08%, QUL Al-Nufais 13.13%, phone tlog 22.02%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 13.08,
      "qul_alnufais": 13.13,
      "tlog_holdout": 22.02
    },
    "sample_count": 600,
    "engine_label": "cohere-transcribe-03-2026",
    "data_license": "Apache-2.0"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1411,
    "raw_value": 14.11,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/deepgram-nova3.json",
    "note": "Row 'Deepgram nova-3' (offline). Deepgram listen file API, model nova-3, language ar. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 14.11% (CER 7.66%, alef-insensitive WER 12.76%). Subsets: EveryAyah 12.39%, QUL Al-Nufais 10.86%, phone tlog 22.09%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 12.39,
      "qul_alnufais": 10.86,
      "tlog_holdout": 22.09
    },
    "sample_count": 600,
    "engine_label": "Deepgram nova-3",
    "data_license": "Apache-2.0"
  },
  {
    "model": "speechmatics-enhanced",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1952,
    "raw_value": 19.52,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/speechmatics-enhanced.json",
    "note": "Row 'Speechmatics enhanced' (offline). Speechmatics batch job API, enhanced operating point, language ar. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 19.52% (CER 8.15%, alef-insensitive WER 18.76%). Subsets: EveryAyah 14.13%, QUL Al-Nufais 21.36%, phone tlog 25.31%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 14.13,
      "qul_alnufais": 21.36,
      "tlog_holdout": 25.31
    },
    "sample_count": 600,
    "engine_label": "Speechmatics enhanced",
    "data_license": "Apache-2.0"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.3371,
    "raw_value": 33.71,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/Qwen3-ASR-1.7B.json",
    "note": "Row 'Qwen3-ASR-1.7B' (offline). hf_repo Qwen/Qwen3-ASR-1.7B, re-run from scratch, language ar. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 33.71% (CER 12.15%, alef-insensitive WER 30.48%). Subsets: EveryAyah 30.83%, QUL Al-Nufais 31.98%, phone tlog 41.16%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 30.83,
      "qul_alnufais": 31.98,
      "tlog_holdout": 41.16
    },
    "sample_count": 600,
    "engine_label": "Qwen3-ASR-1.7B",
    "data_license": "Apache-2.0"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "shmyrev-ru-11set-avg-wer",
    "score": 0.1485,
    "raw_value": 14.85,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/ESpeech/russian_asr_leaderboard/blob/cc62862c6f7d12840458f868a17a3058da7e639f/index.html",
    "source_id": "speech-stt-ru-shmyrev-espeech-mit",
    "by": "independent",
    "date": "2026-05-07",
    "date_basis": "page 'Дата теста: 07.05.2026' (test date); Space revision committed 2026-07-20",
    "note": "Row 'Nemo Parakeet TDT V3': mean WER 14.85% over 11 Russian test sets (Shmyrev benchmark). NeMo 'Parakeet TDT V3' = nvidia/parakeet-tdt-0.6b-v3 (the only Parakeet TDT v3 checkpoint; multilingual incl. Russian).",
    "engine_label": "Nemo Parakeet TDT V3",
    "components": {
      "Аудиокниги АЦ": 6.9,
      "Ru Librispeech": 10.6,
      "CommonVoice 12.0": 5.3,
      "Golos Crowd": 3.9,
      "Golos Farfield": 7.6,
      "Sova устройство": 16.2,
      "Телевещание": 18,
      "Медицина": 13.4,
      "Команды Яндекса": 19.5,
      "Звонки заказы": 32.5,
      "Звонки поддержка": 29.4
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3",
    "bench": "shmyrev-ru-11set-avg-wer",
    "score": 0.1544,
    "raw_value": 15.44,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/ESpeech/russian_asr_leaderboard/blob/cc62862c6f7d12840458f868a17a3058da7e639f/index.html",
    "source_id": "speech-stt-ru-shmyrev-espeech-mit",
    "by": "independent",
    "date": "2026-05-07",
    "date_basis": "page 'Дата теста: 07.05.2026' (test date); Space revision committed 2026-07-20",
    "note": "Row 'Whisper Large V3 Transformers': mean WER 15.44% over 11 Russian test sets (Shmyrev benchmark). Whisper large-v3 run through Hugging Face Transformers (openai/whisper-large-v3).",
    "engine_label": "Whisper Large V3 Transformers",
    "components": {
      "Аудиокниги АЦ": 5.8,
      "Ru Librispeech": 9.5,
      "CommonVoice 12.0": 5.5,
      "Golos Crowd": 14.7,
      "Golos Farfield": 17.6,
      "Sova устройство": 15.9,
      "Телевещание": 17.9,
      "Медицина": 13.8,
      "Команды Яндекса": 18.6,
      "Звонки заказы": 23.7,
      "Звонки поддержка": 26.8
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "shmyrev-ru-11set-avg-wer",
    "score": 0.1615,
    "raw_value": 16.15,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/ESpeech/russian_asr_leaderboard/blob/cc62862c6f7d12840458f868a17a3058da7e639f/index.html",
    "source_id": "speech-stt-ru-shmyrev-espeech-mit",
    "by": "independent",
    "date": "2026-05-07",
    "date_basis": "page 'Дата теста: 07.05.2026' (test date); Space revision committed 2026-07-20",
    "note": "Row 'Whisper v3 Turbo': mean WER 16.15% over 11 Russian test sets (Shmyrev benchmark). Whisper large-v3-turbo (openai/whisper-large-v3-turbo).",
    "engine_label": "Whisper v3 Turbo",
    "components": {
      "Аудиокниги АЦ": 6.5,
      "Ru Librispeech": 9.7,
      "CommonVoice 12.0": 6.2,
      "Golos Crowd": 14.5,
      "Golos Farfield": 18.7,
      "Sova устройство": 16,
      "Телевещание": 18.2,
      "Медицина": 13.7,
      "Команды Яндекса": 21.8,
      "Звонки заказы": 24.8,
      "Звонки поддержка": 27.5
    },
    "data_license": "MIT"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1338,
    "raw_value": 13.38,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Qwen3-ASR-1.7B",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Qwen3-ASR-1.7B' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1338 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id Qwen/Qwen3-ASR-1.7B. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.093,
      "semscore_0_100": 74.1,
      "wer_parkinsons": 0.073,
      "wer_als": 0.176,
      "wer_down_syndrome": 0.227,
      "wer_cerebral_palsy": 0.333
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1388,
    "raw_value": 13.88,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Voxtral-Small-24B",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Voxtral-Small-24B' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1388 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id mistralai/Voxtral-Small-24B-2507. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.098,
      "semscore_0_100": 73.3,
      "wer_parkinsons": 0.075,
      "wer_als": 0.183,
      "wer_down_syndrome": 0.257,
      "wer_cerebral_palsy": 0.347
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1443,
    "raw_value": 14.43,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Whisper-large-v3",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Whisper-large-v3' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1443 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id openai/whisper-large-v3. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.102,
      "semscore_0_100": 72.5,
      "wer_parkinsons": 0.083,
      "wer_als": 0.182,
      "wer_down_syndrome": 0.26,
      "wer_cerebral_palsy": 0.348
    }
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1448,
    "raw_value": 14.48,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Cohere Transcribe",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Cohere Transcribe' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1448 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id CohereLabs/cohere-transcribe-03-2026. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.102,
      "semscore_0_100": 73.1,
      "wer_parkinsons": 0.076,
      "wer_als": 0.191,
      "wer_down_syndrome": 0.289,
      "wer_cerebral_palsy": 0.365
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1531,
    "raw_value": 15.31,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Voxtral-Mini-3B",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Voxtral-Mini-3B' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1531 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id mistralai/Voxtral-Mini-3B-2507. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.106,
      "semscore_0_100": 72.1,
      "wer_parkinsons": 0.087,
      "wer_als": 0.199,
      "wer_down_syndrome": 0.282,
      "wer_cerebral_palsy": 0.366
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.1283,
    "raw_value": 12.83,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'large-v3', hf_id openai/whisper-large-v3. Overall corpus WER 0.1283 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.1501; CER 0.0704; KDHO 0.1134. Run by the leaderboard maintainer.",
    "engine_label": "large-v3",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.0936,
      "fleurs_tr": 0.0606,
      "khanacademy_turkish": 0.0832,
      "mediaspeech_tr": 0.116,
      "real_turnturk": 0.5593,
      "voxforge_tr": 0.0603,
      "youtube_video_transkriptleri_tr": 0.1253
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.1292,
    "raw_value": 12.92,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'large-v3-turbo', hf_id openai/whisper-large-v3-turbo. Overall corpus WER 0.1292 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.151; CER 0.0691; KDHO 0.1144. Run by the leaderboard maintainer.",
    "engine_label": "large-v3-turbo",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.1057,
      "fleurs_tr": 0.0644,
      "khanacademy_turkish": 0.0826,
      "mediaspeech_tr": 0.1143,
      "real_turnturk": 0.5657,
      "voxforge_tr": 0.0697,
      "youtube_video_transkriptleri_tr": 0.1235
    },
    "data_license": "MIT"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.1585,
    "raw_value": 15.85,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'qwen3-asr-1.7b', hf_id Qwen/Qwen3-ASR-1.7B-hf, Qwen's official Transformers-native release of the Qwen3-ASR-1.7B checkpoint (same model, converted format). Overall corpus WER 0.1585 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.1771; CER 0.0715; KDHO 0.1364. Run by the leaderboard maintainer.",
    "engine_label": "qwen3-asr-1.7b",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.1319,
      "fleurs_tr": 0.0819,
      "khanacademy_turkish": 0.1148,
      "mediaspeech_tr": 0.1548,
      "real_turnturk": 0.5799,
      "voxforge_tr": 0.0937,
      "youtube_video_transkriptleri_tr": 0.1529
    },
    "data_license": "MIT"
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/elevenlabs_scribe_v2.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "ElevenLabs Scribe v2",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "elevenlabs-scribe-v2",
    "bench": "voicecodebench-batch-wer",
    "score": 0.069,
    "raw_value": 6.9,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'ElevenLabs Scribe v2': WER 6.9%, CTEM 91.6%, TSR 67.7% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='scribe_v2' provider=elevenlabs endpoint=https://api.elevenlabs.io/v1/speech-to-text mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 91.6,
      "tsr_percent": 67.7
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/elevenlabs_scribe_v2.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "ElevenLabs Scribe v2",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "elevenlabs-scribe-v2",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.916,
    "raw_value": 91.6,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'ElevenLabs Scribe v2': WER 6.9%, CTEM 91.6%, TSR 67.7% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='scribe_v2' provider=elevenlabs endpoint=https://api.elevenlabs.io/v1/speech-to-text mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 6.9,
      "tsr_percent": 67.7
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/deepgram_nova3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Deepgram Nova-3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "deepgram-nova-3",
    "bench": "voicecodebench-batch-wer",
    "score": 0.082,
    "raw_value": 8.2,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Deepgram Nova-3': WER 8.2%, CTEM 90.9%, TSR 68.7% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='nova-3' provider=deepgram endpoint=https://api.deepgram.com/v1/listen mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 90.9,
      "tsr_percent": 68.7
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/deepgram_nova3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Deepgram Nova-3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "deepgram-nova-3",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.909,
    "raw_value": 90.9,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Deepgram Nova-3': WER 8.2%, CTEM 90.9%, TSR 68.7% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='nova-3' provider=deepgram endpoint=https://api.deepgram.com/v1/listen mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 8.2,
      "tsr_percent": 68.7
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-09-01",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/openai_gpt_transcribe.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "OpenAI gpt-transcribe",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "gpt-transcribe",
    "bench": "voicecodebench-batch-wer",
    "score": 0.061,
    "raw_value": 6.1,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'OpenAI gpt-transcribe': WER 6.1%, CTEM 89.3%, TSR 61.0% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='gpt-transcribe' provider=openai endpoint=https://api.openai.com/v1/audio/transcriptions mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 89.3,
      "tsr_percent": 61
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-09-01",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/openai_gpt_transcribe.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "OpenAI gpt-transcribe",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "gpt-transcribe",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.893,
    "raw_value": 89.3,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'OpenAI gpt-transcribe': WER 6.1%, CTEM 89.3%, TSR 61.0% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='gpt-transcribe' provider=openai endpoint=https://api.openai.com/v1/audio/transcriptions mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 6.1,
      "tsr_percent": 61
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/google_cloud_chirp_3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Google Cloud Chirp 3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "google-chirp-3",
    "bench": "voicecodebench-batch-wer",
    "score": 0.058,
    "raw_value": 5.8,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Google Cloud Chirp 3': WER 5.8%, CTEM 88.8%, TSR 60.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='chirp_3' provider=google_cloud endpoint=Google Cloud Speech-to-Text v2 recognize mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 88.8,
      "tsr_percent": 60.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/google_cloud_chirp_3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Google Cloud Chirp 3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "google-chirp-3",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.888,
    "raw_value": 88.8,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Google Cloud Chirp 3': WER 5.8%, CTEM 88.8%, TSR 60.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='chirp_3' provider=google_cloud endpoint=Google Cloud Speech-to-Text v2 recognize mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 5.8,
      "tsr_percent": 60.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/whisper_large_v3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Whisper large-v3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "whisper-large-v3",
    "bench": "voicecodebench-batch-wer",
    "score": 0.058,
    "raw_value": 5.8,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Whisper large-v3': WER 5.8%, CTEM 87.6%, TSR 54.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='whisper-large-v3' provider=groq endpoint=https://api.groq.com/openai/v1/audio/transcriptions mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 87.6,
      "tsr_percent": 54.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/whisper_large_v3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "Whisper large-v3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "whisper-large-v3",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.876,
    "raw_value": 87.6,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'Whisper large-v3': WER 5.8%, CTEM 87.6%, TSR 54.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='whisper-large-v3' provider=groq endpoint=https://api.groq.com/openai/v1/audio/transcriptions mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 5.8,
      "tsr_percent": 54.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-07-24",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/modal_nvidia_parakeet_tdt_0_6b_v3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "NVIDIA Parakeet TDT 0.6B v3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "voicecodebench-batch-wer",
    "score": 0.093,
    "raw_value": 9.3,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'NVIDIA Parakeet TDT 0.6B v3': WER 9.3%, CTEM 78.6%, TSR 37.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='nvidia/parakeet-tdt-0.6b-v3' provider=modal endpoint=https://tyler-5928--voice-code-bench-parakeet-parakeettranscribe-462ac5.modal.run mode=batch (HF revision b51b7dc0fbf7f266a97880fb4b626c56d28f4b96). Stored: WER.",
    "components": {
      "ctem_percent": 78.6,
      "tsr_percent": 37.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-07-24",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/modal_nvidia_parakeet_tdt_0_6b_v3.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "NVIDIA Parakeet TDT 0.6B v3",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "voicecodebench-batch-ctem",
    "score": 0.786,
    "raw_value": 78.6,
    "scale_divisor": 100,
    "note": "Table 5 (v2) batch row 'NVIDIA Parakeet TDT 0.6B v3': WER 9.3%, CTEM 78.6%, TSR 37.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='nvidia/parakeet-tdt-0.6b-v3' provider=modal endpoint=https://tyler-5928--voice-code-bench-parakeet-parakeettranscribe-462ac5.modal.run mode=batch (HF revision b51b7dc0fbf7f266a97880fb4b626c56d28f4b96). Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 9.3,
      "tsr_percent": 37.3
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "africodeswitch-en-yo-cap2000-wer",
    "score": 0.661,
    "raw_value": 66.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.11786v1",
    "source_id": "speech-stt-yoruba-cs-report",
    "by": "independent",
    "date": "2026-09-10",
    "date_basis": "arXiv v1 submission date (10 Sep 2026); run date not stated",
    "engine_label": "Parakeet-TDT-0.6B-v2",
    "sample_count": 2000,
    "data_license": "CC BY 4.0",
    "note": "Table II row 'Parakeet-TDT-0.6B-v2' (family ASR): WER 66.1% on the cap2000 manifest (2,000 English-Yoruba code-switched utterances from the AfriCodeSwitch validation corpus), zero-shot greedy decoding, NFC normalisation, case-folding, punctuation stripping, diacritic-sensitive. Identity: exact checkpoint name as printed in Table II and the §V model list. English-token error 34.4%, Yoruba-token error 99.0% (near-total failure on Yoruba for all ASR systems). Other metrics in components.",
    "components": {
      "cer_percent": 35.7,
      "wer_diacritic_insensitive_percent": 66,
      "en_token_er_percent": 34.4,
      "yo_token_er_percent": 99,
      "switch_error_seter_percent": 68.9,
      "sper_at1_percent": 67.2,
      "sper_at3_percent": 65.2
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "africodeswitch-en-yo-cap2000-wer",
    "score": 0.676,
    "raw_value": 67.6,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.11786v1",
    "source_id": "speech-stt-yoruba-cs-report",
    "by": "independent",
    "date": "2026-09-10",
    "date_basis": "arXiv v1 submission date (10 Sep 2026); run date not stated",
    "engine_label": "Qwen3-ASR-1.7B",
    "sample_count": 2000,
    "data_license": "CC BY 4.0",
    "note": "Table II row 'Qwen3-ASR-1.7B' (family ASR): WER 67.6% on the cap2000 manifest (2,000 English-Yoruba code-switched utterances from the AfriCodeSwitch validation corpus), zero-shot greedy decoding, NFC normalisation, case-folding, punctuation stripping, diacritic-sensitive. Identity: exact checkpoint name as printed in Table II and the §V model list. English-token error 37.9%, Yoruba-token error 99.2% (near-total failure on Yoruba for all ASR systems). Other metrics in components.",
    "components": {
      "cer_percent": 39.6,
      "wer_diacritic_insensitive_percent": 67.9,
      "en_token_er_percent": 37.9,
      "yo_token_er_percent": 99.2,
      "switch_error_seter_percent": 70.4,
      "sper_at1_percent": 68.2,
      "sper_at3_percent": 66
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "africodeswitch-en-yo-cap2000-wer",
    "score": 0.7,
    "raw_value": 70,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.11786v1",
    "source_id": "speech-stt-yoruba-cs-report",
    "by": "independent",
    "date": "2026-09-10",
    "date_basis": "arXiv v1 submission date (10 Sep 2026); run date not stated",
    "engine_label": "Canary-Qwen-2.5B",
    "sample_count": 2000,
    "data_license": "CC BY 4.0",
    "note": "Table II row 'Canary-Qwen-2.5B' (family ASR): WER 70.0% on the cap2000 manifest (2,000 English-Yoruba code-switched utterances from the AfriCodeSwitch validation corpus), zero-shot greedy decoding, NFC normalisation, case-folding, punctuation stripping, diacritic-sensitive. Identity: exact checkpoint name as printed in Table II and the §V model list. English-token error 33.5%, Yoruba-token error 99.4% (near-total failure on Yoruba for all ASR systems). Other metrics in components.",
    "components": {
      "cer_percent": 40.1,
      "wer_diacritic_insensitive_percent": 70,
      "en_token_er_percent": 33.5,
      "yo_token_er_percent": 99.4,
      "switch_error_seter_percent": 68.7,
      "sper_at1_percent": 66.8,
      "sper_at3_percent": 64.8
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "africodeswitch-en-yo-cap2000-wer",
    "score": 0.712,
    "raw_value": 71.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.11786v1",
    "source_id": "speech-stt-yoruba-cs-report",
    "by": "independent",
    "date": "2026-09-10",
    "date_basis": "arXiv v1 submission date (10 Sep 2026); run date not stated",
    "engine_label": "Whisper-large-v3",
    "sample_count": 2000,
    "data_license": "CC BY 4.0",
    "note": "Table II row 'Whisper-large-v3' (family ASR): WER 71.2% on the cap2000 manifest (2,000 English-Yoruba code-switched utterances from the AfriCodeSwitch validation corpus), zero-shot greedy decoding, NFC normalisation, case-folding, punctuation stripping, diacritic-sensitive. Whisper run with forced English (the auto-language row, 74.8%, is not used). Identity: exact checkpoint name as printed in Table II and the §V model list. English-token error 36.8%, Yoruba-token error 98.9% (near-total failure on Yoruba for all ASR systems). Other metrics in components.",
    "components": {
      "cer_percent": 41.7,
      "wer_diacritic_insensitive_percent": 70.9,
      "en_token_er_percent": 36.8,
      "yo_token_er_percent": 98.9,
      "switch_error_seter_percent": 70.2,
      "sper_at1_percent": 68,
      "sper_at3_percent": 66
    }
  },
  {
    "model": "granite-speech-4-1-2b",
    "bench": "africodeswitch-en-yo-cap2000-wer",
    "score": 0.735,
    "raw_value": 73.5,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.11786v1",
    "source_id": "speech-stt-yoruba-cs-report",
    "by": "independent",
    "date": "2026-09-10",
    "date_basis": "arXiv v1 submission date (10 Sep 2026); run date not stated",
    "engine_label": "Granite-Speech-4.1-2B",
    "sample_count": 2000,
    "data_license": "CC BY 4.0",
    "note": "Table II row 'Granite-Speech-4.1-2B' (family ASR): WER 73.5% on the cap2000 manifest (2,000 English-Yoruba code-switched utterances from the AfriCodeSwitch validation corpus), zero-shot greedy decoding, NFC normalisation, case-folding, punctuation stripping, diacritic-sensitive. Identity: label 'Granite-Speech-4.1-2B' taken as the base checkpoint ibm-granite/granite-speech-4.1-2b (not the -nar or -plus variants). English-token error 38.9%, Yoruba-token error 99.4% (near-total failure on Yoruba for all ASR systems). Other metrics in components.",
    "components": {
      "cer_percent": 42.8,
      "wer_diacritic_insensitive_percent": 74,
      "en_token_er_percent": 38.9,
      "yo_token_er_percent": 99.4,
      "switch_error_seter_percent": 72.9,
      "sper_at1_percent": 69.6,
      "sper_at3_percent": 67.2
    }
  },
  {
    "model": "cartesia-sonic-3",
    "bench": "rime-supportbench-pref-vs-coda",
    "score": 0.449968,
    "raw_value": -0.200127,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/supportbench/supportbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'cartesia_sonic3_default' of results/supportbench/supportbench_results.json: mean_rime_minus_competitor 0.200127 (95% CI 0.1113..0.2879), 750 scripts, 2730 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: Cartesia API model 'sonic-3' (api_version 2026-03-01), voice e07c00bc-4134-4eae-9ea4-1a55fb45746b.",
    "sample_count": 750,
    "judgments": 2730,
    "engine_label": "cartesia_sonic3_default",
    "ci95_raw": [
      -0.287877,
      -0.111259
    ]
  },
  {
    "model": "deepgram-aura-2",
    "bench": "rime-supportbench-pref-vs-coda",
    "score": 0.407968,
    "raw_value": -0.368127,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/supportbench/supportbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'deepgram_aura2_thalia' of results/supportbench/supportbench_results.json: mean_rime_minus_competitor 0.368127 (95% CI 0.2852..0.4490), 750 scripts, 2730 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: Deepgram model 'aura-2-thalia-en' (Aura-2, thalia voice).",
    "sample_count": 750,
    "judgments": 2730,
    "engine_label": "deepgram_aura2_thalia",
    "ci95_raw": [
      -0.448964,
      -0.285171
    ]
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "rime-supportbench-pref-vs-coda",
    "score": 0.469571,
    "raw_value": -0.121714,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/supportbench/supportbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'elevenlabs_flash25_cindycasualnarrator' of results/supportbench/supportbench_results.json: mean_rime_minus_competitor 0.121714 (95% CI 0.0513..0.1973), 750 scripts, 5250 valid judgments, Holm p 0.00018; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: ElevenLabs model 'eleven_flash_v2_5', voice vCHG6sKIqAbXWNNm5vpY, stability 0.5, similarity_boost 0.75, text normalization on.",
    "sample_count": 750,
    "judgments": 5250,
    "engine_label": "elevenlabs_flash25_cindycasualnarrator",
    "ci95_raw": [
      -0.1973,
      -0.051317
    ]
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "rime-supportbench-pref-vs-coda",
    "score": 0.274905,
    "raw_value": -0.900381,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/supportbench/supportbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'openai_4ominitts_coral' of results/supportbench/supportbench_results.json: mean_rime_minus_competitor 0.900381 (95% CI 0.8320..0.9686), 750 scripts, 2730 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: OpenAI model 'gpt-4o-mini-tts', voice coral.",
    "sample_count": 750,
    "judgments": 2730,
    "engine_label": "openai_4ominitts_coral",
    "ci95_raw": [
      -0.968576,
      -0.832036
    ]
  },
  {
    "model": "cartesia-sonic-3",
    "bench": "rime-contentbench-pref-vs-coda",
    "score": 0.449714,
    "raw_value": -0.201143,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/contentbench/contentbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'cartesia_sonic3_default' of results/contentbench/contentbench_results.json: mean_rime_minus_competitor 0.201143 (95% CI 0.1178..0.2795), 400 scripts, 2240 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: Cartesia API model 'sonic-3' (api_version 2026-03-01), voice e07c00bc-4134-4eae-9ea4-1a55fb45746b.",
    "sample_count": 400,
    "judgments": 2240,
    "engine_label": "cartesia_sonic3_default",
    "ci95_raw": [
      -0.27951,
      -0.1178
    ]
  },
  {
    "model": "deepgram-aura-2",
    "bench": "rime-contentbench-pref-vs-coda",
    "score": 0.538375,
    "raw_value": 0.1535,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/contentbench/contentbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'deepgram_aura2_thalia' of results/contentbench/contentbench_results.json: mean_rime_minus_competitor -0.153500 (95% CI -0.2290..-0.0763), 400 scripts, 2240 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: Deepgram model 'aura-2-thalia-en' (Aura-2, thalia voice).",
    "sample_count": 400,
    "judgments": 2240,
    "engine_label": "deepgram_aura2_thalia",
    "ci95_raw": [
      0.076269,
      0.229047
    ]
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "rime-contentbench-pref-vs-coda",
    "score": 0.541036,
    "raw_value": 0.164143,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/contentbench/contentbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'elevenlabs_flash25_cindycasualnarrator' of results/contentbench/contentbench_results.json: mean_rime_minus_competitor -0.164143 (95% CI -0.2566..-0.0715), 400 scripts, 2240 valid judgments, Holm p 0.00021; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: ElevenLabs model 'eleven_flash_v2_5', voice vCHG6sKIqAbXWNNm5vpY, stability 0.5, similarity_boost 0.75, text normalization on.",
    "sample_count": 400,
    "judgments": 2240,
    "engine_label": "elevenlabs_flash25_cindycasualnarrator",
    "ci95_raw": [
      0.071473,
      0.256607
    ]
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "rime-contentbench-pref-vs-coda",
    "score": 0.422375,
    "raw_value": -0.3105,
    "raw_unit": "mean paired score, competitor minus Rime Coda (-2..+2)",
    "scale_offset": -2,
    "scale_divisor": 4,
    "source": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/contentbench/contentbench_results.json",
    "source_id": "speech-rime-alpha-bench-ccby",
    "by": "independent",
    "date": "2026-09-13",
    "date_basis": "results commit 99d850a (2026-09-13) over the export snapshot 3b65260; listener collection finished before the 2026-09-12 specification freeze (exact dates not published)",
    "note": "Row 'openai_4ominitts_coral' of results/contentbench/contentbench_results.json: mean_rime_minus_competitor 0.310500 (95% CI 0.2147..0.4135), 400 scripts, 2240 valid judgments, Holm p 2e-05; sign flipped to competitor-minus-Rime. Identity from configs/provider_configs.json: OpenAI model 'gpt-4o-mini-tts', voice coral.",
    "sample_count": 400,
    "judgments": 2240,
    "engine_label": "openai_4ominitts_coral",
    "ci95_raw": [
      -0.413549,
      -0.214678
    ]
  },
  {
    "model": "elevenlabs-v3",
    "bench": "voicefirst-indic10-overall-bt",
    "score": 1056.28,
    "raw_value": 1056.28,
    "raw_unit": "Bradley–Terry rating (Elo-like)",
    "source": "https://arxiv.org/html/2604.21481v2",
    "source_id": "speech-voicefirst-indic-ccby",
    "by": "independent",
    "date": "2026-04-23",
    "date_basis": "arXiv v1 submission 2026-04-23 (v2 2026-06-23 pinned); audio generated after 2026-03-04",
    "note": "Table 3 row 'Eleven Labs v3' (rank 2): score 1056.28 ± 2 (95% bootstrap CI), 40,800 comparisons, win rate 57%, 9 languages. Identity: ElevenLabs v3 (eleven_v3); sentences partly generated with Gemini-3-pro-preview (blog accessed 2026-03-04), so synthesis post-dates the Feb-2026 v3 GA; default voice, non-streaming.",
    "sample_count": 40800,
    "engine_label": "Eleven Labs v3"
  },
  {
    "model": "cartesia-sonic-3",
    "bench": "voicefirst-indic10-overall-bt",
    "score": 1050.83,
    "raw_value": 1050.83,
    "raw_unit": "Bradley–Terry rating (Elo-like)",
    "source": "https://arxiv.org/html/2604.21481v2",
    "source_id": "speech-voicefirst-indic-ccby",
    "by": "independent",
    "date": "2026-04-23",
    "date_basis": "arXiv v1 submission 2026-04-23 (v2 2026-06-23 pinned); audio generated after 2026-03-04",
    "note": "Table 3 row 'Sonic 3' (rank 2): score 1050.83 ± 3 (95% bootstrap CI), 33,795 comparisons, win rate 56%, 8 languages. Identity: Cartesia Sonic 3 (sonic-3), distinct from Sonic 3.5/3.6; default voice, non-streaming.",
    "sample_count": 33795,
    "engine_label": "Sonic 3"
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "voicefirst-indic10-overall-bt",
    "score": 993.94,
    "raw_value": 993.94,
    "raw_unit": "Bradley–Terry rating (Elo-like)",
    "source": "https://arxiv.org/html/2604.21481v2",
    "source_id": "speech-voicefirst-indic-ccby",
    "by": "independent",
    "date": "2026-04-23",
    "date_basis": "arXiv v1 submission 2026-04-23 (v2 2026-06-23 pinned); audio generated after 2026-03-04",
    "note": "Table 3 row 'Speech 2.8 HD' (rank 5): score 993.94 ± 6 (95% bootstrap CI), 7,834 comparisons, win rate 47%, 2 languages. Identity: MiniMax Speech 2.8 HD; default voice, non-streaming.",
    "sample_count": 7834,
    "engine_label": "Speech 2.8 HD"
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "voicefirst-indic10-overall-bt",
    "score": 942.76,
    "raw_value": 942.76,
    "raw_unit": "Bradley–Terry rating (Elo-like)",
    "source": "https://arxiv.org/html/2604.21481v2",
    "source_id": "speech-voicefirst-indic-ccby",
    "by": "independent",
    "date": "2026-04-23",
    "date_basis": "arXiv v1 submission 2026-04-23 (v2 2026-06-23 pinned); audio generated after 2026-03-04",
    "note": "Table 3 row 'GPT-4o-mini TTS' (rank 6): score 942.76 ± 4 (95% bootstrap CI), 15,207 comparisons, win rate 40%, 5 languages. Identity: OpenAI gpt-4o-mini-tts; default voice, non-streaming.",
    "sample_count": 15207,
    "engine_label": "GPT-4o-mini TTS"
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "instructttseval-en-avg",
    "score": 0.887,
    "raw_value": 88.7,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 5 (EN subset) row 'gemini-flash*': APS 92.3, DSD 93.8, RP 80.1, Avg. 88.7. Identity: Section 4.2: gemini-2.5-flash-preview-tts (gemini-flash); voice picked from the female/male list per caption. Gemini-as-a-judge.",
    "engine_label": "gemini-flash*",
    "components": {
      "APS": 92.3,
      "DSD": 93.8,
      "RP": 80.1
    }
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "instructttseval-en-avg",
    "score": 0.803,
    "raw_value": 80.3,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 5 (EN subset) row 'gemini-pro*': APS 87.6, DSD 86.0, RP 67.2, Avg. 80.3. Identity: Section 4.2: gemini-2.5-pro-preview-tts (gemini-pro). Gemini-as-a-judge.",
    "engine_label": "gemini-pro*",
    "components": {
      "APS": 87.6,
      "DSD": 86,
      "RP": 67.2
    }
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "instructttseval-en-avg",
    "score": 0.685,
    "raw_value": 68.5,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 5 (EN subset) row 'gpt-4o-mini-tts': APS 76.4, DSD 74.3, RP 54.8, Avg. 68.5. Identity: Section 4.2: gpt-4o-mini-tts. Gemini-as-a-judge.",
    "engine_label": "gpt-4o-mini-tts",
    "components": {
      "APS": 76.4,
      "DSD": 74.3,
      "RP": 54.8
    }
  },
  {
    "model": "hume-octave",
    "bench": "instructttseval-en-avg",
    "score": 0.711,
    "raw_value": 71.1,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 5 (EN subset) row 'hume*': APS 83.0, DSD 75.3, RP 54.3, Avg. 71.1. Identity: Section 4.2: 'Hume' (Hume AI text-to-speech with free-text voice description); evaluated before 2025-06-19, when Octave (1) was Hume's only TTS model — Octave 2 launched Oct 2025. Gemini-as-a-judge.",
    "engine_label": "hume*",
    "components": {
      "APS": 83,
      "DSD": 75.3,
      "RP": 54.3
    }
  },
  {
    "model": "gemini-2-5-flash-preview-tts",
    "bench": "instructttseval-zh-avg",
    "score": 0.854,
    "raw_value": 85.4,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 6 (ZH subset) row 'gemini-flash*': APS 88.2, DSD 90.9, RP 77.3, Avg. 85.4. Identity: Section 4.2: gemini-2.5-flash-preview-tts (gemini-flash); voice picked from the female/male list per caption. Gemini-as-a-judge.",
    "engine_label": "gemini-flash*",
    "components": {
      "APS": 88.2,
      "DSD": 90.9,
      "RP": 77.3
    }
  },
  {
    "model": "gemini-2-5-pro-preview-tts",
    "bench": "instructttseval-zh-avg",
    "score": 0.848,
    "raw_value": 84.8,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 6 (ZH subset) row 'gemini-pro*': APS 89.0, DSD 90.1, RP 75.5, Avg. 84.8. Identity: Section 4.2: gemini-2.5-pro-preview-tts (gemini-pro). Gemini-as-a-judge.",
    "engine_label": "gemini-pro*",
    "components": {
      "APS": 89,
      "DSD": 90.1,
      "RP": 75.5
    }
  },
  {
    "model": "gpt-4o-mini-tts",
    "bench": "instructttseval-zh-avg",
    "score": 0.511,
    "raw_value": 51.1,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 6 (ZH subset) row 'gpt-4o-mini-tts': APS 54.9, DSD 52.3, RP 46.0, Avg. 51.1. Identity: Section 4.2: gpt-4o-mini-tts. Gemini-as-a-judge.",
    "engine_label": "gpt-4o-mini-tts",
    "components": {
      "APS": 54.9,
      "DSD": 52.3,
      "RP": 46
    }
  },
  {
    "model": "indextts-2",
    "bench": "sap-voicerecon-bws-reconstruction",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "IndexTTS2",
    "sample_count": 193,
    "score": 1167.29,
    "raw_value": 0.963,
    "raw_unit": "Plackett–Luce log-worth relative to original recording",
    "note": "Table 2 row 'IndexTTS2 [ 32 ]': BWS intel. 1.921, BWS recon. 0.963, WER 9.9%, PER 52.4%, UTMOS 2.88, Spk. Sim. 0.670, TTSDS|SAP 86.9, TTSDS|LibriTTS 85.3, TTSDS mean 86.1. Identity: IndexTTS2 (cites arXiv:2506.21619; the only released IndexTTS2 checkpoint is IndexTeam/IndexTTS-2)."
  },
  {
    "model": "xtts-v2",
    "bench": "sap-voicerecon-bws-reconstruction",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "XTTS(v2)",
    "sample_count": 193,
    "score": 776.08,
    "raw_value": -1.289,
    "raw_unit": "Plackett–Luce log-worth relative to original recording",
    "note": "Table 2 row 'XTTS(v2) [ 46 ]': BWS intel. 0.490, BWS recon. -1.289, WER 17.3%, PER 58.1%, UTMOS 2.63, Spk. Sim. 0.404, TTSDS|SAP 86.0, TTSDS|LibriTTS 80.9, TTSDS mean 83.5. Identity: printed label 'XTTS(v2)'."
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "sap-voicerecon-bws-reconstruction",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MetaVoice",
    "sample_count": 193,
    "score": 795.71,
    "raw_value": -1.176,
    "raw_unit": "Plackett–Luce log-worth relative to original recording",
    "note": "Table 2 row 'MetaVoice [ 45 ]': BWS intel. -1.076, BWS recon. -1.176, WER 43.3%, PER 76.8%, UTMOS 2.09, Spk. Sim. 0.418, TTSDS|SAP 80.9, TTSDS|LibriTTS 77.6, TTSDS mean 79.2. Identity: MetaVoice (github.com/metavoiceio/metavoice-src; its only released model is metavoice-1B-v0.1)."
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "sap-voicerecon-bws-reconstruction",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "TorToiSe",
    "sample_count": 193,
    "score": 856.16,
    "raw_value": -0.828,
    "raw_unit": "Plackett–Luce log-worth relative to original recording",
    "note": "Table 2 row 'TorToiSe [ 43 ]': BWS intel. 1.940, BWS recon. -0.828, WER 15.1%, PER 48.4%, UTMOS 3.15, Spk. Sim. 0.372, TTSDS|SAP 82.5, TTSDS|LibriTTS 85.7, TTSDS mean 84.1. Identity: TorToiSe (Betker 2023; released weights are tortoise-tts-v2)."
  },
  {
    "model": "indextts-2",
    "bench": "sap-voicerecon-wer",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "IndexTTS2",
    "sample_count": 193,
    "score": 0.099,
    "raw_value": 9.9,
    "raw_unit": "percent",
    "scale_divisor": 100,
    "note": "Table 2 row 'IndexTTS2 [ 32 ]': BWS intel. 1.921, BWS recon. 0.963, WER 9.9%, PER 52.4%, UTMOS 2.88, Spk. Sim. 0.670, TTSDS|SAP 86.9, TTSDS|LibriTTS 85.3, TTSDS mean 86.1. Identity: IndexTTS2 (cites arXiv:2506.21619; the only released IndexTTS2 checkpoint is IndexTeam/IndexTTS-2)."
  },
  {
    "model": "xtts-v2",
    "bench": "sap-voicerecon-wer",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "XTTS(v2)",
    "sample_count": 193,
    "score": 0.173,
    "raw_value": 17.3,
    "raw_unit": "percent",
    "scale_divisor": 100,
    "note": "Table 2 row 'XTTS(v2) [ 46 ]': BWS intel. 0.490, BWS recon. -1.289, WER 17.3%, PER 58.1%, UTMOS 2.63, Spk. Sim. 0.404, TTSDS|SAP 86.0, TTSDS|LibriTTS 80.9, TTSDS mean 83.5. Identity: printed label 'XTTS(v2)'."
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "sap-voicerecon-wer",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MetaVoice",
    "sample_count": 193,
    "score": 0.433,
    "raw_value": 43.3,
    "raw_unit": "percent",
    "scale_divisor": 100,
    "note": "Table 2 row 'MetaVoice [ 45 ]': BWS intel. -1.076, BWS recon. -1.176, WER 43.3%, PER 76.8%, UTMOS 2.09, Spk. Sim. 0.418, TTSDS|SAP 80.9, TTSDS|LibriTTS 77.6, TTSDS mean 79.2. Identity: MetaVoice (github.com/metavoiceio/metavoice-src; its only released model is metavoice-1B-v0.1)."
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "sap-voicerecon-wer",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "TorToiSe",
    "sample_count": 193,
    "score": 0.151,
    "raw_value": 15.1,
    "raw_unit": "percent",
    "scale_divisor": 100,
    "note": "Table 2 row 'TorToiSe [ 43 ]': BWS intel. 1.940, BWS recon. -0.828, WER 15.1%, PER 48.4%, UTMOS 3.15, Spk. Sim. 0.372, TTSDS|SAP 82.5, TTSDS|LibriTTS 85.7, TTSDS mean 84.1. Identity: TorToiSe (Betker 2023; released weights are tortoise-tts-v2)."
  },
  {
    "model": "indextts-2",
    "bench": "sap-voicerecon-ttsds-mean",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "IndexTTS2",
    "sample_count": 193,
    "score": 0.861,
    "raw_value": 86.1,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100,
    "note": "Table 2 row 'IndexTTS2 [ 32 ]': BWS intel. 1.921, BWS recon. 0.963, WER 9.9%, PER 52.4%, UTMOS 2.88, Spk. Sim. 0.670, TTSDS|SAP 86.9, TTSDS|LibriTTS 85.3, TTSDS mean 86.1. Identity: IndexTTS2 (cites arXiv:2506.21619; the only released IndexTTS2 checkpoint is IndexTeam/IndexTTS-2)."
  },
  {
    "model": "xtts-v2",
    "bench": "sap-voicerecon-ttsds-mean",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "XTTS(v2)",
    "sample_count": 193,
    "score": 0.835,
    "raw_value": 83.5,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100,
    "note": "Table 2 row 'XTTS(v2) [ 46 ]': BWS intel. 0.490, BWS recon. -1.289, WER 17.3%, PER 58.1%, UTMOS 2.63, Spk. Sim. 0.404, TTSDS|SAP 86.0, TTSDS|LibriTTS 80.9, TTSDS mean 83.5. Identity: printed label 'XTTS(v2)'."
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "sap-voicerecon-ttsds-mean",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MetaVoice",
    "sample_count": 193,
    "score": 0.792,
    "raw_value": 79.2,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100,
    "note": "Table 2 row 'MetaVoice [ 45 ]': BWS intel. -1.076, BWS recon. -1.176, WER 43.3%, PER 76.8%, UTMOS 2.09, Spk. Sim. 0.418, TTSDS|SAP 80.9, TTSDS|LibriTTS 77.6, TTSDS mean 79.2. Identity: MetaVoice (github.com/metavoiceio/metavoice-src; its only released model is metavoice-1B-v0.1)."
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "sap-voicerecon-ttsds-mean",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "TorToiSe",
    "sample_count": 193,
    "score": 0.841,
    "raw_value": 84.1,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100,
    "note": "Table 2 row 'TorToiSe [ 43 ]': BWS intel. 1.940, BWS recon. -0.828, WER 15.1%, PER 48.4%, UTMOS 3.15, Spk. Sim. 0.372, TTSDS|SAP 82.5, TTSDS|LibriTTS 85.7, TTSDS mean 84.1. Identity: TorToiSe (Betker 2023; released weights are tortoise-tts-v2)."
  },
  {
    "model": "openaudio-s1-mini",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.82275,
    "raw_value": 4.291,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'OpenAudio s1 mini [ 31 ]': S-NAT 4.291 ± 0.053, S-SPK-SIM 4.158 ± 0.078, S-ACC-SIM 4.125 ± 0.081, O-WER 0.093, O-SPK-SIM 0.617, O-UTMOS 4.344. This column: 4.291 ± 0.053 (95% CI). Identity: printed 'OpenAudio s1 mini' (reference is the Fish-Speech paper).",
    "engine_label": "OpenAudio s1 mini",
    "sample_count": 160,
    "ci95_raw": [
      4.238,
      4.344
    ]
  },
  {
    "model": "chatterbox",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.8075,
    "raw_value": 4.23,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'Chatterbox [ 39 ]': S-NAT 4.230 ± 0.058, S-SPK-SIM 4.333 ± 0.071, S-ACC-SIM 4.094 ± 0.085, O-WER 0.328, O-SPK-SIM 0.708, O-UTMOS 4.212. This column: 4.23 ± 0.058 (95% CI). Identity: Resemble AI Chatterbox-TTS (github.com/resemble-ai/chatterbox), English VCTK prompts; mapped to the original English Chatterbox, not Chatterbox Multilingual/Turbo.",
    "engine_label": "Chatterbox",
    "sample_count": 160,
    "ci95_raw": [
      4.172,
      4.288
    ]
  },
  {
    "model": "indextts-2",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.74425,
    "raw_value": 3.977,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'IndexTTS2 [ 38 ]': S-NAT 3.977 ± 0.068, S-SPK-SIM 4.530 ± 0.058, S-ACC-SIM 4.248 ± 0.081, O-WER 0.034, O-SPK-SIM 0.743, O-UTMOS 4.046. This column: 3.977 ± 0.068 (95% CI). Identity: IndexTTS2 (arXiv:2506.21619).",
    "engine_label": "IndexTTS2",
    "sample_count": 160,
    "ci95_raw": [
      3.909,
      4.045
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.73475,
    "raw_value": 3.939,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'TorToiSe [ 27 ]': S-NAT 3.939 ± 0.070, S-SPK-SIM 3.696 ± 0.090, S-ACC-SIM 3.462 ± 0.100, O-WER 0.122, O-SPK-SIM 0.449, O-UTMOS 4.079. This column: 3.939 ± 0.070 (95% CI). Identity: TorToiSe (Betker 2023; released weights tortoise-tts-v2).",
    "engine_label": "TorToiSe",
    "sample_count": 160,
    "ci95_raw": [
      3.869,
      4.009
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.7095,
    "raw_value": 3.838,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'XTTS [ 28 ]': S-NAT 3.838 ± 0.067, S-SPK-SIM 3.694 ± 0.090, S-ACC-SIM 3.745 ± 0.090, O-WER 0.559, O-SPK-SIM 0.609, O-UTMOS 3.997. This column: 3.838 ± 0.067 (95% CI). Identity: XTTS (Casanova et al., Interspeech 2024 — the paper describing the released XTTS-v2 model).",
    "engine_label": "XTTS",
    "sample_count": 160,
    "ci95_raw": [
      3.771,
      3.905
    ]
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.51825,
    "raw_value": 3.073,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'MetaVoice [ 35 ]': S-NAT 3.073 ± 0.083, S-SPK-SIM 4.060 ± 0.075, S-ACC-SIM 4.047 ± 0.076, O-WER 3.399, O-SPK-SIM 0.561, O-UTMOS 3.546. This column: 3.073 ± 0.083 (95% CI). Identity: MetaVoice (metavoice-src; only released model metavoice-1B-v0.1).",
    "engine_label": "MetaVoice",
    "sample_count": 160,
    "ci95_raw": [
      2.99,
      3.156
    ]
  },
  {
    "model": "openaudio-s1-mini",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.7895,
    "raw_value": 4.158,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'OpenAudio s1 mini [ 31 ]': S-NAT 4.291 ± 0.053, S-SPK-SIM 4.158 ± 0.078, S-ACC-SIM 4.125 ± 0.081, O-WER 0.093, O-SPK-SIM 0.617, O-UTMOS 4.344. This column: 4.158 ± 0.078 (95% CI). Identity: printed 'OpenAudio s1 mini' (reference is the Fish-Speech paper).",
    "engine_label": "OpenAudio s1 mini",
    "sample_count": 160,
    "ci95_raw": [
      4.08,
      4.236
    ]
  },
  {
    "model": "chatterbox",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.83325,
    "raw_value": 4.333,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'Chatterbox [ 39 ]': S-NAT 4.230 ± 0.058, S-SPK-SIM 4.333 ± 0.071, S-ACC-SIM 4.094 ± 0.085, O-WER 0.328, O-SPK-SIM 0.708, O-UTMOS 4.212. This column: 4.333 ± 0.071 (95% CI). Identity: Resemble AI Chatterbox-TTS (github.com/resemble-ai/chatterbox), English VCTK prompts; mapped to the original English Chatterbox, not Chatterbox Multilingual/Turbo.",
    "engine_label": "Chatterbox",
    "sample_count": 160,
    "ci95_raw": [
      4.262,
      4.404
    ]
  },
  {
    "model": "indextts-2",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.8825,
    "raw_value": 4.53,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'IndexTTS2 [ 38 ]': S-NAT 3.977 ± 0.068, S-SPK-SIM 4.530 ± 0.058, S-ACC-SIM 4.248 ± 0.081, O-WER 0.034, O-SPK-SIM 0.743, O-UTMOS 4.046. This column: 4.53 ± 0.058 (95% CI). Identity: IndexTTS2 (arXiv:2506.21619).",
    "engine_label": "IndexTTS2",
    "sample_count": 160,
    "ci95_raw": [
      4.472,
      4.588
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.674,
    "raw_value": 3.696,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'TorToiSe [ 27 ]': S-NAT 3.939 ± 0.070, S-SPK-SIM 3.696 ± 0.090, S-ACC-SIM 3.462 ± 0.100, O-WER 0.122, O-SPK-SIM 0.449, O-UTMOS 4.079. This column: 3.696 ± 0.090 (95% CI). Identity: TorToiSe (Betker 2023; released weights tortoise-tts-v2).",
    "engine_label": "TorToiSe",
    "sample_count": 160,
    "ci95_raw": [
      3.606,
      3.786
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.6735,
    "raw_value": 3.694,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'XTTS [ 28 ]': S-NAT 3.838 ± 0.067, S-SPK-SIM 3.694 ± 0.090, S-ACC-SIM 3.745 ± 0.090, O-WER 0.559, O-SPK-SIM 0.609, O-UTMOS 3.997. This column: 3.694 ± 0.090 (95% CI). Identity: XTTS (Casanova et al., Interspeech 2024 — the paper describing the released XTTS-v2 model).",
    "engine_label": "XTTS",
    "sample_count": 160,
    "ci95_raw": [
      3.604,
      3.784
    ]
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.765,
    "raw_value": 4.06,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'MetaVoice [ 35 ]': S-NAT 3.073 ± 0.083, S-SPK-SIM 4.060 ± 0.075, S-ACC-SIM 4.047 ± 0.076, O-WER 3.399, O-SPK-SIM 0.561, O-UTMOS 3.546. This column: 4.06 ± 0.075 (95% CI). Identity: MetaVoice (metavoice-src; only released model metavoice-1B-v0.1).",
    "engine_label": "MetaVoice",
    "sample_count": 160,
    "ci95_raw": [
      3.985,
      4.135
    ]
  },
  {
    "model": "zoom-scribe-v1",
    "bench": "open-asr-wer",
    "score": 0.0547,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.47%; entry zoom/scribe_v1; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "granite-4-0-1b-speech",
    "bench": "open-asr-wer",
    "score": 0.0552,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.52%; entry ibm-granite/granite-4.0-1b-speech; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "granite-speech-3-3-8b",
    "bench": "open-asr-wer",
    "score": 0.0574,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 5.74%; entry ibm-granite/granite-speech-3.3-8b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "granite-speech-3-3-2b",
    "bench": "open-asr-wer",
    "score": 0.06,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6%; entry ibm-granite/granite-speech-3.3-2b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "phi-4-multimodal-instruct",
    "bench": "open-asr-wer",
    "score": 0.0602,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.02%; entry microsoft/Phi-4-multimodal-instruct; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "assemblyai-universal-3-pro",
    "bench": "open-asr-wer",
    "score": 0.0621,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.21%; entry assemblyai/universal-3-pro; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "aquavoice-avalon-v1-en",
    "bench": "open-asr-wer",
    "score": 0.0624,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.24%; entry aquavoice/avalon-v1-en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "canary-1b-flash",
    "bench": "open-asr-wer",
    "score": 0.0635,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.35%; entry nvidia/canary-1b-flash; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "google-chirp-2",
    "bench": "open-asr-wer",
    "score": 0.0642,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.42%; entry google/chirp_2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "open-asr-wer",
    "score": 0.0642,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.42%; entry Qwen/Qwen3-ASR-0.6B; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "canary-1b",
    "bench": "open-asr-wer",
    "score": 0.065038,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.50375%; entry nvidia/canary-1b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "moonshine-streaming-medium",
    "bench": "open-asr-wer",
    "score": 0.0666,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.66%; entry usefulsensors/moonshine-streaming-medium; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "crisperwhisper",
    "bench": "open-asr-wer",
    "score": 0.0667,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.67%; entry nyrahealth/CrisperWhisper; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "zipformer-transducer-xl-290m",
    "bench": "open-asr-wer",
    "score": 0.0679,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "lab",
    "date": "2026-05-13",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.79%; entry soundsgoodai/Zipformer-transducer-XL-290M; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default; row added after the 26 March snapshot by the model maker's submission (commit 6a4f34c, PR #155 'Add SoundsgoodAI Zipformer XL submission', with its own soundsgoodai/run_eval.py), hence by=lab and date = commit date",
    "source_id": "open-asr"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "open-asr-wer",
    "score": 0.0688,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 6.88%; entry elevenlabs/scribe_v1; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-1-1b",
    "bench": "open-asr-wer",
    "score": 0.07015,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.015%; entry nvidia/parakeet-tdt-1.1b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "glm-asr-nano-2512",
    "bench": "open-asr-wer",
    "score": 0.0703,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.03%; entry zai-org/GLM-ASR-Nano-2512; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "canary-180m-flash",
    "bench": "open-asr-wer",
    "score": 0.0712,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.12%; entry nvidia/canary-180m-flash; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default; CAUTION: this row's 8 per-set WER columns average 6.94%, not 7.12% - the published 'Avg. WER' is used as for every other row",
    "source_id": "open-asr"
  },
  {
    "model": "revai-fusion",
    "bench": "open-asr-wer",
    "score": 0.0712,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.12%; entry revai/fusion; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "lite-whisper-large-v3-acc",
    "bench": "open-asr-wer",
    "score": 0.0723,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.23%; entry efficient-speech/lite-whisper-large-v3-acc; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-ctc-1-1b",
    "bench": "open-asr-wer",
    "score": 0.073988,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.39875%; entry nvidia/parakeet-ctc-1.1b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "owsm-ctc-v4-1b",
    "bench": "open-asr-wer",
    "score": 0.0742,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.42%; entry espnet/owsm_ctc_v4_1B; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "lite-whisper-large-v3",
    "bench": "open-asr-wer",
    "score": 0.0743,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.43%; entry efficient-speech/lite-whisper-large-v3; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-ctc-110m",
    "bench": "open-asr-wer",
    "score": 0.0749,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.49%; entry nvidia/parakeet-tdt_ctc-110m; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-rnnt-0-6b",
    "bench": "open-asr-wer",
    "score": 0.07495,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.495%; entry nvidia/parakeet-rnnt-0.6b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v3",
    "bench": "open-asr-wer",
    "score": 0.075175,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.5175%; entry distil-whisper/distil-large-v3; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-ctc-0-6b",
    "bench": "open-asr-wer",
    "score": 0.0769,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.69%; entry nvidia/parakeet-ctc-0.6b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "open-asr-wer",
    "score": 0.0777,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.77%; entry microsoft/VibeVoice-ASR-HF; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "lite-whisper-large-v3-turbo-acc",
    "bench": "open-asr-wer",
    "score": 0.0777,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.77%; entry efficient-speech/lite-whisper-large-v3-turbo-acc; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default; CAUTION: this row's 8 per-set WER columns average 7.52%, not 7.77% - the published 'Avg. WER' is used as for every other row",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v2",
    "bench": "open-asr-wer",
    "score": 0.07835,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.835%; entry openai/whisper-large-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "moonshine-streaming-small",
    "bench": "open-asr-wer",
    "score": 0.0784,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.84%; entry usefulsensors/moonshine-streaming-small; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v2",
    "bench": "open-asr-wer",
    "score": 0.079212,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.92125%; entry distil-whisper/distil-large-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large",
    "bench": "open-asr-wer",
    "score": 0.079425,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.9425%; entry openai/whisper-large; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "revai-machine",
    "bench": "open-asr-wer",
    "score": 0.0795,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 7.95%; entry revai/machine; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-medium-en",
    "bench": "open-asr-wer",
    "score": 0.0809,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.09%; entry openai/whisper-medium.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "owsm-ctc-v3-1-1b",
    "bench": "open-asr-wer",
    "score": 0.0812,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.12%; entry espnet/owsm_ctc_v3.1_1B; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-7b-v2",
    "bench": "open-asr-wer",
    "score": 0.0814,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.14%; entry facebook/omniASR-LLM-7B-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "lite-whisper-large-v3-fast",
    "bench": "open-asr-wer",
    "score": 0.0816,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.16%; entry efficient-speech/lite-whisper-large-v3-fast; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "owsm-ctc-v3-2-ft-1b",
    "bench": "open-asr-wer",
    "score": 0.0825,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.25%; entry espnet/owsm_ctc_v3.2_ft_1B; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "google-chirp",
    "bench": "open-asr-wer",
    "score": 0.0829,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.29%; entry google/chirp; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-ctc-large",
    "bench": "open-asr-wer",
    "score": 0.083188,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.31875%; entry nvidia/stt_en_conformer_ctc_large; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "speechbrain-asr-conformer-largescaleasr",
    "bench": "open-asr-wer",
    "score": 0.0848,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.48%; entry speechbrain/asr-conformer-largescaleasr; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "ultravox-v0-6-llama-3-1-8b",
    "bench": "open-asr-wer",
    "score": 0.0852,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.52%; entry ultravox/fixie-ai-ultravox-v0_6-llama-3_1-8b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "distil-small-en",
    "bench": "open-asr-wer",
    "score": 0.085675,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.5675%; entry distil-whisper/distil-small.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-small-en",
    "bench": "open-asr-wer",
    "score": 0.0859,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.59%; entry openai/whisper-small.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "distil-medium-en",
    "bench": "open-asr-wer",
    "score": 0.08765,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.765%; entry distil-whisper/distil-medium.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-fastconformer-ctc-large",
    "bench": "open-asr-wer",
    "score": 0.08965,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 8.965%; entry nvidia/stt_en_fastconformer_ctc_large; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-fastconformer-transducer-large",
    "bench": "open-asr-wer",
    "score": 0.090575,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 9.0575%; entry nvidia/stt_en_fastconformer_transducer_large; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "moonshine-base",
    "bench": "open-asr-wer",
    "score": 0.0999,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 9.99%; entry usefulsensors/moonshine-base; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-transducer-small",
    "bench": "open-asr-wer",
    "score": 0.102575,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 10.2575%; entry nvidia/stt_en_conformer_transducer_small; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-base-en",
    "bench": "open-asr-wer",
    "score": 0.103175,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 10.3175%; entry openai/whisper-base.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "abr-asr-19m-v2-en",
    "bench": "open-asr-wer",
    "score": 0.1061,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 10.61%; entry abr-ai/asr-19m-v2-en-32b; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-7b-v2",
    "bench": "open-asr-wer",
    "score": 0.1065,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 10.65%; entry facebook/omniASR-CTC-7B-v2; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-ctc-small",
    "bench": "open-asr-wer",
    "score": 0.111587,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 11.15875%; entry nvidia/stt_en_conformer_ctc_small; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "moonshine-streaming-tiny",
    "bench": "open-asr-wer",
    "score": 0.12,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 12%; entry usefulsensors/moonshine-streaming-tiny; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "moonshine-tiny",
    "bench": "open-asr-wer",
    "score": 0.1265,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 12.65%; entry usefulsensors/moonshine-tiny; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-tiny-en",
    "bench": "open-asr-wer",
    "score": 0.128063,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 12.80625%; entry openai/whisper-tiny.en; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "speechbrain-asr-wav2vec2-librispeech",
    "bench": "open-asr-wer",
    "score": 0.143475,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 14.3475%; entry speechbrain/asr-wav2vec2-librispeech; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "open-asr-wer",
    "score": 0.212725,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 21.2725%; entry facebook/wav2vec2-large-960h-lv60-self; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "mms-1b-all",
    "bench": "open-asr-wer",
    "score": 0.225388,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 22.53875%; entry facebook/mms-1b-all; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "hubert-xlarge-ls960-ft",
    "bench": "open-asr-wer",
    "score": 0.225475,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 22.5475%; entry facebook/hubert-xlarge-ls960-ft; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "hubert-large-ls960-ft",
    "bench": "open-asr-wer",
    "score": 0.226938,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 22.69375%; entry facebook/hubert-large-ls960-ft; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-large-robust-ft-libri-960h",
    "bench": "open-asr-wer",
    "score": 0.229312,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 22.93125%; entry facebook/wav2vec2-large-robust-ft-libri-960h; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "data2vec-audio-large-960h",
    "bench": "open-asr-wer",
    "score": 0.2321,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 23.21%; entry facebook/data2vec-audio-large-960h; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-conformer-rope-large-960h-ft",
    "bench": "open-asr-wer",
    "score": 0.232838,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 23.28375%; entry facebook/wav2vec2-conformer-rope-large-960h-ft; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-conformer-rel-pos-large-960h-ft",
    "bench": "open-asr-wer",
    "score": 0.232913,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 23.29125%; entry facebook/wav2vec2-conformer-rel-pos-large-960h-ft; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "open-asr-wer",
    "score": 0.267725,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 26.7725%; entry facebook/wav2vec2-large-960h; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "data2vec-audio-base-960h",
    "bench": "open-asr-wer",
    "score": 0.283025,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 28.3025%; entry facebook/data2vec-audio-base-960h; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "wav2vec2-base-960h",
    "bench": "open-asr-wer",
    "score": 0.294025,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 29.4025%; entry facebook/wav2vec2-base-960h; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "mms-1b-fl102",
    "bench": "open-asr-wer",
    "score": 0.397963,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/16743c6dabc69abcb8fd5bf529a95e3c6c16ac3c/scripts/data/en_shortform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_shortform.csv @ 16743c6 (committed 2026-05-22; README: 'CSV results as of 26 March 2026'); column 'Avg. WER' = 39.79625%; entry facebook/mms-1b-fl102; mean of 8 public short-form sets (AMI, Earnings22, Gigaspeech, LS clean, LS other, SPGISpeech, Tedlium, Voxpopuli) - older mix than the current Space default",
    "source_id": "open-asr"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "open-asr-longform-wer",
    "score": 0.058967,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 5.896666667 (mean of long-form earnings21 6.35, earnings22 9.03, tedlium 2.31); entry elevenlabs/scribe_v1",
    "source_id": "open-asr"
  },
  {
    "model": "assemblyai-universal-3-pro",
    "bench": "open-asr-longform-wer",
    "score": 0.068433,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 6.843333333 (mean of long-form earnings21 7.62, earnings22 10.59, tedlium 2.32); entry assembly/universal-3-pro",
    "source_id": "open-asr"
  },
  {
    "model": "revai-fusion",
    "bench": "open-asr-longform-wer",
    "score": 0.085167,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 8.516666667 (mean of long-form earnings21 7.56, earnings22 15.47, tedlium 2.52); entry revai/fusion",
    "source_id": "open-asr"
  },
  {
    "model": "revai-machine",
    "bench": "open-asr-longform-wer",
    "score": 0.087567,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 8.756666667 (mean of long-form earnings21 7.63, earnings22 15.72, tedlium 2.92); entry revai/machine",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v3",
    "bench": "open-asr-longform-wer",
    "score": 0.0984,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.84 (mean of long-form earnings21 10.58, earnings22 14.98, tedlium 3.96); entry distil-whisper/distil-large-v3",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large",
    "bench": "open-asr-longform-wer",
    "score": 0.0941,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.41 (mean of long-form earnings21 10.66, earnings22 14.11, tedlium 3.46); entry openai/whisper-large",
    "source_id": "open-asr"
  },
  {
    "model": "whisper-large-v2",
    "bench": "open-asr-longform-wer",
    "score": 0.094467,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.446666667 (mean of long-form earnings21 10.9, earnings22 14.24, tedlium 3.2); entry openai/whisper-large-v2",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-ctc-1-1b",
    "bench": "open-asr-longform-wer",
    "score": 0.092767,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.276666667 (mean of long-form earnings21 10.14, earnings22 14.46, tedlium 3.23); entry nvidia/parakeet-ctc-1.1b",
    "source_id": "open-asr"
  },
  {
    "model": "google-chirp",
    "bench": "open-asr-longform-wer",
    "score": 0.1154,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 11.54 (mean of long-form earnings21 14.29, earnings22 14.81, tedlium 5.52); entry google/chirp",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-ctc-0-6b",
    "bench": "open-asr-longform-wer",
    "score": 0.097733,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.773333333 (mean of long-form earnings21 10.79, earnings22 14.97, tedlium 3.56); entry nvidia/parakeet-ctc-0.6b",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-transducer-small",
    "bench": "open-asr-longform-wer",
    "score": 0.137433,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 13.74333333 (mean of long-form earnings21 15.05, earnings22 20.72, tedlium 5.46); entry nvidia/stt_en_conformer_transducer_small",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-ctc-large",
    "bench": "open-asr-longform-wer",
    "score": 0.111,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 11.1 (mean of long-form earnings21 12.11, earnings22 17.51, tedlium 3.68); entry nvidia/stt_en_conformer_ctc_large",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-rnnt-0-6b",
    "bench": "open-asr-longform-wer",
    "score": 0.095033,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 9.503333333 (mean of long-form earnings21 11.13, earnings22 14.14, tedlium 3.24); entry nvidia/parakeet-rnnt-0.6b",
    "source_id": "open-asr"
  },
  {
    "model": "parakeet-tdt-1-1b",
    "bench": "open-asr-longform-wer",
    "score": 0.101867,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 10.18666667 (mean of long-form earnings21 11.57, earnings22 14.98, tedlium 4.01); entry nvidia/parakeet-tdt-1.1b",
    "source_id": "open-asr"
  },
  {
    "model": "distil-large-v2",
    "bench": "open-asr-longform-wer",
    "score": 0.146867,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 14.68666667 (mean of long-form earnings21 14.97, earnings22 19.24, tedlium 9.85); entry distil-whisper/distil-large-v2",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-ctc-small",
    "bench": "open-asr-longform-wer",
    "score": 0.152833,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 15.28333333 (mean of long-form earnings21 16.44, earnings22 23.15, tedlium 6.26); entry nvidia/stt_en_conformer_ctc_small",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-fastconformer-ctc-large",
    "bench": "open-asr-longform-wer",
    "score": 0.155233,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 15.52333333 (mean of long-form earnings21 17.02, earnings22 24.29, tedlium 5.26); entry nvidia/stt_en_fastconformer_ctc_large",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-conformer-transducer-large",
    "bench": "open-asr-longform-wer",
    "score": 0.153967,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 15.39666667 (mean of long-form earnings21 17.26, earnings22 23.69, tedlium 5.24); entry nvidia/stt_en_conformer_transducer_large",
    "source_id": "open-asr"
  },
  {
    "model": "stt-en-fastconformer-transducer-large",
    "bench": "open-asr-longform-wer",
    "score": 0.162867,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 16.28666667 (mean of long-form earnings21 18.15, earnings22 25.3, tedlium 5.41); entry nvidia/stt_en_fastconformer_transducer_large",
    "source_id": "open-asr"
  },
  {
    "model": "distil-medium-en",
    "bench": "open-asr-longform-wer",
    "score": 0.1896,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/en_longform.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/en_longform.csv @ 090df12 (2026-03-26); column 'Avg (without CORAAL)' = 18.96 (mean of long-form earnings21 19.48, earnings22 24.82, tedlium 12.58); entry distil-whisper/distil-medium.en",
    "source_id": "open-asr"
  },
  {
    "model": "assemblyai-universal-3-pro",
    "bench": "open-asr-multilingual-wer",
    "score": 0.032315,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 3.231538462 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry assembly/universal-3-pro",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-7b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0439,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.39 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-7B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "phi-4-multimodal-instruct",
    "bench": "open-asr-multilingual-wer",
    "score": 0.044077,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.407692308 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry microsoft/Phi-4-multimodal-instruct",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.046838,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 4.683846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-7B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-7b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.058415,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 5.841538462 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-7B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-3b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0594,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 5.94 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-3B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-1b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.060892,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.089230769 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-1B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-3b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.061654,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.165384615 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-3B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-7b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.061777,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.177692308 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-7B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-3b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.062569,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.256923077 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-3B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-1b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0667,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 6.67 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-1B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-3b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.0705,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 7.05 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-3B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-1b-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.079638,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 7.963846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-1B-v2",
    "source_id": "open-asr"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.082377,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 8.237692308 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry Qwen/Qwen3-ASR-0.6B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-300m-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.084231,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 8.423076923 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-300M-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-1b",
    "bench": "open-asr-multilingual-wer",
    "score": 0.087015,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 8.701538462 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-1B",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-llm-300m",
    "bench": "open-asr-multilingual-wer",
    "score": 0.092538,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 9.253846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-LLM-300M",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-300m-v2",
    "bench": "open-asr-multilingual-wer",
    "score": 0.143438,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 14.34384615 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-300M-v2",
    "source_id": "open-asr"
  },
  {
    "model": "omniasr-ctc-300m",
    "bench": "open-asr-multilingual-wer",
    "score": 0.149285,
    "source": "https://github.com/huggingface/open_asr_leaderboard/blob/090df1224774bb3c152b57ed3f99898e679222e5/scripts/data/multilingual.csv",
    "by": "independent",
    "date": "2026-03-26",
    "note": "Open ASR Leaderboard results CSV, github.com/huggingface/open_asr_leaderboard (Apache-2.0); scripts/data/multilingual.csv @ 090df12 (2026-03-26); column 'Avg' = 14.92846154 (mean over 13 sets: CoVoST2/MLS/FLEURS for de, fr, it, es, pt); entry facebook/omniASR-CTC-300M",
    "source_id": "open-asr"
  },
  {
    "model": "claude-2-0",
    "bench": "gpqa-diamond",
    "score": 0.3465909090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-2.0'; run id mTa4f52zsEuxdrPy7WfQvX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025076255983551475.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-2-0",
    "bench": "math-level-5",
    "score": 0.1172583081570997,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-2.0'; run id Gs4vxvDD5wmrSvyTkqx2YV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006595670913969979.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-2-0",
    "bench": "otis-mock-aime",
    "score": 0.025,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-2.0'; run id 7LLTmqs4LC5GLVfJEqJWSC; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015179897658722631.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-2-1",
    "bench": "gpqa-diamond",
    "score": 0.32954545454545453,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-2.1'; run id eMAisrJd7fAUHPfcPWXeXo; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02324885083265289.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-2-1",
    "bench": "otis-mock-aime",
    "score": 0.019444444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-2.1'; run id 2vg9qw9pd8zSyRMu9YNiVm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010477515876628662.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "gpqa-diamond",
    "score": 0.3813131313131313,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-5-haiku-20241022'; run id JpLxrpvthx7A75sbaHZ9Et; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026351515390511556.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "math-level-5",
    "score": 0.46355740181268884,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-5-haiku-20241022'; run id GSvi9Zf5aK3t5TfCPqNRHy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011611728105526228.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "otis-mock-aime",
    "score": 0.043055555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-5-haiku-20241022'; run id cqhrR5KnwpcuAVpHkZGQFd; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016237638827815495.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "gpqa-diamond",
    "score": 0.5404040404040404,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20240620'; run id 78Wpvp8WDByAcQRBas72cs; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027528555453348653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "math-level-5",
    "score": 0.5168051359516617,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20240620'; run id H42HJG29bwqzaMewTMsmky; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0116250120233614.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "otis-mock-aime",
    "score": 0.06527777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20240620'; run id 2xNPhKLsiWLHQBLLDGU4VJ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022293131815660016.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "gpqa-diamond",
    "score": 0.553030303030303,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20241022'; run id JCCVcVLz3J4TQXSU67hMJZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028208090147078847.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "math-level-5",
    "score": 0.5694864048338368,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20241022'; run id AjroyuZECXLtMTwnzaMHtX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011091284283577038.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "otis-mock-aime",
    "score": 0.08472222222222223,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-5-sonnet-20241022'; run id geVVaRWxuprKUqnW2VzvCa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026363956855902478.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "gpqa-diamond",
    "score": 0.7973484848484849,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-7-sonnet-20250219_64K'; run id 2t26VSNdsAE5pefQEMP25x; effort 64K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026506883123060064.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "math-level-5",
    "score": 0.9116314199395771,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-13",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-7-sonnet-20250219_64K'; run id epCyFNTf6vyUi5Dby7GeTq; effort 64K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007803306377082018.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "otis-mock-aime",
    "score": 0.5777777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-13",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-7-sonnet-20250219_64K'; run id QW3FUBn5qtAFEX4oQuzEAV; effort 64K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07446027270295805.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "swe-bench-verified",
    "score": 0.609504132231405,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-04",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-3-7-sonnet-20250219'; run id cnM8U6moeqrp74wnXRj2Jj; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022198466971122266. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "gpqa-diamond",
    "score": 0.3630050505050505,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-haiku-20240307'; run id T3FWdCCWBryVjxwTcoY5bK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024180910672074975.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "math-level-5",
    "score": 0.1487915407854985,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-haiku-20240307'; run id QSnWVKUVj9six4jYj4HJD9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007229965913719499.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "otis-mock-aime",
    "score": 0.018055555555555554,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-haiku-20240307'; run id ThMd6DNDTtyWxhPcb9Kq4y; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008098544298396251.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "epoch-chess",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-3-opus-20240229'; run id QHfG7caFexs6vKJfmYsVBp; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "gpqa-diamond",
    "score": 0.4715909090909091,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-opus-20240229'; run id hTnDa4gn8RYduCEYqoNywe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026395393956533515.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "math-level-5",
    "score": 0.37481117824773413,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-opus-20240229'; run id 6SNzvpd6kXXxqpJoaHsSDd; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01058728426376393.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "otis-mock-aime",
    "score": 0.04722222222222222,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-opus-20240229'; run id mYb9MGQD39RS8VdDyMqujr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019579258477123922.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "simpleqa-verified",
    "score": 0.126,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'claude-3-opus-20240229'; run id NxVp4VF8uQbL5S6hx6HFRB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010499249222407964.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "gpqa-diamond",
    "score": 0.4059343434343434,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-3-sonnet-20240229'; run id QLejPZXhjX6SQCQgzjNvCG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023920223491748827.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "math-level-5",
    "score": 0.18174093655589124,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-3-sonnet-20240229'; run id FD2ztfjFUeqGyrtF8yuCUB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007981817344047291.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "otis-mock-aime",
    "score": 0.025,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-3-sonnet-20240229'; run id AeRCEiG87bMFpxbxbmjCyn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01950297827553939.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "epoch-chess",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-20",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805'; run id cb9qw7m5zpPWKqRDLB4ktU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "epoch-ebr",
    "score": 0.079365,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805'; run id NMncsmkDVG6ZmNufPxNs2m; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.024390243902439025,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805_32K'; run id 9zT72xRR6wZhiwraJTNPyz; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024390243902439022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "frontiermath",
    "score": 0.12631578947368421,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805_32K'; run id 5eRkmXCURH2pRbhJoukfnC; effort 32K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0197127354633578.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "gpqa-diamond",
    "score": 0.7676767676767676,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-08-05",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805_27K'; run id 3obcsGCfuwYeEm9wRyqzPg; effort 27K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030088629490217445.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "epoch-mystery-games",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-07-25",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805_24K'; run id 2AbeCjy9w4KJWKhbKjtFTJ; effort 24K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "otis-mock-aime",
    "score": 0.6888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-08-05",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805_27K'; run id UH9RY8MCAotDgFocHKA4j6; effort 27K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06979205927323111.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "swe-bench-verified",
    "score": 0.7334710743801653,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-1-20250805'; run id cA6Mq67yaynLhY8F5kes3c; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020118245554677085. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "gpqa-diamond",
    "score": 0.7626262626262627,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-20250514_16K'; run id CtxFusPxCraL3EeRJu2c5k; effort 16K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03031371053819892.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "math-level-5",
    "score": 0.850453172205438,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-05-22",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-20250514'; run id YbrpPbWuYAXNqbSfU4so7f; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009804690367660946.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "otis-mock-aime",
    "score": 0.6444444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-05-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-20250514_27K'; run id VrRccga4uRVuCjNp637MeY; effort 27K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0721639236343101.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "swe-bench-verified",
    "score": 0.7066115702479339,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-06",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'claude-opus-4-20250514'; run id RbcvuVChvxK6SMQbPK8iRn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02071754535128056. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "gpqa-diamond",
    "score": 0.7918781725888325,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-20250514_59K'; run id J9wjcQqKJ7hCSYnnoa5E9i; effort 59K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02674821434321096.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "math-level-5",
    "score": 0.8436555891238671,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-05-22",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-20250514'; run id dZLNhNKQVUgnB5ygAEUCcT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00998490304339695.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "otis-mock-aime",
    "score": 0.6888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-05-23",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'claude-sonnet-4-20250514_59K'; run id g8qaq3s2TJqfvrskHYmapM; effort 59K; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06979205927323111.",
    "source_id": "epoch-internal"
  },
  {
    "model": "dbrx-instruct",
    "bench": "gpqa-diamond",
    "score": 0.32891414141414144,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'dbrx-instruct'; run id TTa3hu7PgRxHgizVreZsdJ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03256944333564935.",
    "source_id": "epoch-internal"
  },
  {
    "model": "dbrx-instruct",
    "bench": "math-level-5",
    "score": 0.11650302114803625,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'dbrx-instruct'; run id PMbefrjykQmTBpABMa7FJi; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007355853708151387.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'deepseek-llm-67b-chat'; run id 3t8aBtMCF3Z24rUH9qJePk; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "gpqa-diamond",
    "score": 0.24621212121212122,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'deepseek-llm-67b-chat'; run id 82RTgqgiC2UuM5M37L3WWM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013623351950761947.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "math-level-5",
    "score": 0.06391616314199396,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'deepseek-llm-67b-chat'; run id Cm4UkgA5TtmzQegrq3rbvm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004016844149550666.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "otis-mock-aime",
    "score": 0.008333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'deepseek-llm-67b-chat'; run id Y5rC5REnP83Q7mAoMtjiRq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004700633956814719.",
    "source_id": "epoch-internal"
  },
  {
    "model": "eurus-2-7b-prime",
    "bench": "gpqa-diamond",
    "score": 0.3390151515151515,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Eurus-2-7B-PRIME'; run id RJm4u2vXF6PUmThJGzAfHE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02107317439394748.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "gpqa-diamond",
    "score": 0.33964646464646464,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.0-pro-001'; run id ibpS8bs4z3wD3Hb6F3kpdV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020110933405176737.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "math-level-5",
    "score": 0.11244335347432025,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-1.0-pro-001'; run id iUdecHq9vU5utytzgTXzqY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005990949270731778.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "otis-mock-aime",
    "score": 0.011111111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.0-pro-001'; run id NpJvYQDcmwbGgS5MzZUPUg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006054026310473157.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "gpqa-diamond",
    "score": 0.40372474747474746,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-001'; run id Hc2ycMP6DyCZxCPjKbZFQU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023852475604024478.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "math-level-5",
    "score": 0.2512273413897281,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-001'; run id CPpd4hqQHtZ5pUejBELwKb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009456393246526343.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "otis-mock-aime",
    "score": 0.03888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-001'; run id U9GVBhYjRzU55YVjna5mMT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014363726487520755.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "gpqa-diamond",
    "score": 0.47316919191919193,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-002'; run id PCLvrs4TEBtqQqcYeU8pAG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027737408277040958.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "math-level-5",
    "score": 0.6186744712990937,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-002'; run id kDnfjYKQpnLGEjuhBc9AvM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01120122127580226.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "otis-mock-aime",
    "score": 0.1625,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-002'; run id QdGZ2m52PWt3EkLPbcoRwz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04111375571619442.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-8b-001",
    "bench": "gpqa-diamond",
    "score": 0.32954545454545453,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-8b-001'; run id gFT36W4KU3wggLQ9nNCctg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02416551116211757.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-flash-8b-001",
    "bench": "otis-mock-aime",
    "score": 0.04583333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-flash-8b-001'; run id jGY5Ee5P9dyStA4Fw5Pa6q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020100756305184247.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "gpqa-diamond",
    "score": 0.4586489898989899,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-001'; run id PKdqpKZ9tqDWyWJ7GksBWz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02617295639558329.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "math-level-5",
    "score": 0.4074773413897281,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-001'; run id PxLmyq7PFfARAuqxg5oPtE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010574902459145802.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "otis-mock-aime",
    "score": 0.06805555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-001'; run id 3S6fjGAMKWKK3zcdxkmh9H; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01893034295514878.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "gpqa-diamond",
    "score": 0.5722853535353535,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-002'; run id 9TrLEEw8TvsyugyCjwvUvD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027959709636504244.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "math-level-5",
    "score": 0.7039274924471299,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-002'; run id QyqLmgcJpeDMsnQyaaJVkR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010174893415515739.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "otis-mock-aime",
    "score": 0.23055555555555557,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-1.5-pro-002'; run id fhCdyX9uUS4WPtjLiVdBdK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05186782011216038.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "gpqa-diamond",
    "score": 0.6414141414141414,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-flash-001'; run id fbszwTApm3f28z5VA6oXBK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027395618179027574.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "math-level-5",
    "score": 0.8216578549848943,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-flash-001'; run id SqwGZz2UfTmbvPiw2rq3LH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008734081505176936.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "otis-mock-aime",
    "score": 0.3111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-flash-001'; run id ZnGSRrKbiEnuju4kk4sgVb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06311218144454454.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-thinking-exp-01-21",
    "bench": "gpqa-diamond",
    "score": 0.5707070707070707,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-flash-thinking-exp-01-21'; run id ag9w2w4xa47ghZaWt5oZey; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03526552724601199.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-thinking-exp-01-21",
    "bench": "otis-mock-aime",
    "score": 0.5777777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-flash-thinking-exp-01-21'; run id cXz2gsCmGYSvQ6Ey4LDiN7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07446027270295805.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-pro-exp-02-05",
    "bench": "gpqa-diamond",
    "score": 0.6565656565656566,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-pro-exp-02-05'; run id g5YE7G6DXb45EH4HxEdiht; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03383201223244441.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-pro-exp-02-05",
    "bench": "math-level-5",
    "score": 0.8345921450151057,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-2.0-pro-exp-02-05'; run id fvDt6HaHV5d3gSEuavixsv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01021492751504924.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "otis-mock-aime",
    "score": 0.7305555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-flash-preview-04-17'; run id KpZYG5nvdJ6QYweLz6dn5a; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05532781604070319.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "otis-mock-aime",
    "score": 0.7083333333333334,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-flash-preview-05-20'; run id 9e62UjoHQbcZD2xjgfRkdK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05715807427254223.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro-exp-03-25",
    "bench": "gpqa-diamond",
    "score": 0.8383838383838383,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-03-31",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro-exp-03-25'; run id WKjzEQdfnBHnYeZoKeCWnB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02622591986362927.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "gpqa-diamond",
    "score": 0.6666666666666666,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-06-03",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro-preview-05-06'; run id TT66gGiVoeYLGQMCESPrEG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.033586181457325226.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "math-level-5",
    "score": 0.9590256797583081,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-05-08",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro-preview-05-06'; run id Mi8LCxmduKukxazAtmuhqg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004221388011004008.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "gpqa-diamond",
    "score": 0.8484848484848485,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-06-05",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-2.5-pro-preview-06-05'; run id Q4qTzHnjLyiCCRvNpXioUq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02554565042660364.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "epoch-chess",
    "score": 0.4,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview_high'; run id HVtbACyemDaTPW5XUDC6Jf; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0492365963917331.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.17073170731707318,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview'; run id ZczEhu69FQ9Cnns8NT6ew4; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05949419959829497.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "frontiermath",
    "score": 0.512280701754386,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview'; run id EtgrDq6ei6kZe8K7QbXUe3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02966059084324674.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "gpqa-diamond",
    "score": 0.8939393939393939,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview_high'; run id gVdraDT3pXmaExP9VbqME4; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021938047738853106.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "epoch-mystery-games",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview_high'; run id MvdunHkVURna3JuyDjhzEc; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "otis-mock-aime",
    "score": 0.9555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview_high'; run id a4uE6KgMhGSUjcnemRnbkA; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.031067790907534743.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "simpleqa-verified",
    "score": 0.668,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview_high'; run id 7cc6cDCXjsPjtKt36bbeMR; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014899597242811567.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "swe-bench-verified",
    "score": 0.7541322314049587,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-18",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3-flash-preview'; run id jxoyMNmTnYirjLJN6wRNmG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019593002139508684. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "epoch-chess",
    "score": 0.31,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gemini-3-pro-preview'; run id Ey65gb5LG3RdtZxMweDfAU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04648231987117317.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "gpqa-diamond",
    "score": 0.9261363636363636,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-11-19",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemini-3-pro-preview'; run id Cg54udQGcTCnfDGX2qePLZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016526963574741.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "otis-mock-aime",
    "score": 0.9138888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-11-19",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemini-3-pro-preview'; run id 4QM4h7ndQiDRL2anjKeddK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03658872186755414.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "swe-bench-verified",
    "score": 0.7293388429752066,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-13",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gemini-3-pro-preview'; run id 5XdYZBG5Wy8DUMRqcYv4fn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02021641171153417. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-27b",
    "bench": "gpqa-diamond",
    "score": 0.3648989898989899,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-2-27b-it'; run id ZUJxsJDpHnLXxNiDcvj22W; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023897884428691632.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-27b",
    "bench": "math-level-5",
    "score": 0.2788897280966767,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemma-2-27b-it'; run id PM3GCQcVK7t2MujNp4F4kH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009939474306804167.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-27b",
    "bench": "otis-mock-aime",
    "score": 0.013888888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-2-27b-it'; run id jD6rJdvfNPErKEz9qkpssj; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00907862525871088.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gpqa-diamond",
    "score": 0.2746212121212121,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gemma-2-9b-it'; run id fQjoHKyUEPAuwdrBGfL8SG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023032467954778795.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-9b",
    "bench": "math-level-5",
    "score": 0.2100641993957704,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gemma-2-9b-it'; run id Hdcqs42VnPNVBquXttexhq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008990724846256171.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemma-2-9b",
    "bench": "otis-mock-aime",
    "score": 0.005555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gemma-2-9b-it'; run id WNE7cveVN8rsi7DmsFsFVy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.003883473863441843.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id JWmZHq9PfAKgNpJJ7FqpGb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "frontiermath",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id FNaBAvmeTiRRFDW63KemZ7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "gpqa-diamond",
    "score": 0.27178030303030304,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id eZaQhpbhbzbo3sbKGS5Efa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01953579968788737.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "math-level-5",
    "score": 0.1163141993957704,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id 9tSgxGV3nzhTBpq7GcX7yx; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005543945913851348.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "epoch-mystery-games",
    "score": 0.03,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id UfmijCsx59sZ8xpcbcWDms; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01714466079977653.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "otis-mock-aime",
    "score": 0.022222222222222223,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-0125'; run id 7ZVhkukwaRXU2Sx2ttibPF; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022222222222222223.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "gpqa-diamond",
    "score": 0.2803030303030303,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-1106'; run id enhRcQmfN6RFB2zu7R3Uxr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018720372561004013.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "math-level-5",
    "score": 0.15889350453172205,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-3.5-turbo-1106'; run id cJWh887zSiTtnxm9XAJVkr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006959408516001194.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "gpqa-diamond",
    "score": 0.42266414141414144,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4-0125-preview'; run id 4j3WRhhruXrfaq9Jki3NDc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026795949166711065.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "math-level-5",
    "score": 0.3541351963746224,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4-0125-preview'; run id c4V7sc8TUNJ8wQKRhuA2Rv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010855237562085883.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0314",
    "bench": "gpqa-diamond",
    "score": 0.3573232323232323,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-23",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4-0314'; run id FkbLkW5ed8rnZGFJMevDFn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02363099375599328.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0314",
    "bench": "otis-mock-aime",
    "score": 0.005555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4-0314'; run id cLTgetxaaTCZoA9vwY4SdL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.003883473863441843.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0613",
    "bench": "epoch-chess",
    "score": 0.04,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4-0613'; run id QWSuwo8dwFP8QiSGbjQ33G; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01969463855669324.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0613",
    "bench": "gpqa-diamond",
    "score": 0.30650252525252525,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4-0613'; run id fu7pWqbSC4MUBdwv3GFNa5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02337322948270623.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0613",
    "bench": "math-level-5",
    "score": 0.22970166163141995,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4-0613'; run id WsSrH8oHBm7m7vq9yTRqSC; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008465534027077483.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0613",
    "bench": "epoch-mystery-games",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4-0613'; run id CvujZuqfj5r7qsa52Smyx9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.032659863237109045.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-0613",
    "bench": "otis-mock-aime",
    "score": 0.011111111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-23",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4-0613'; run id hGVxg4EhSSJSCdH88m9soq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005362818328685681.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "gpqa-diamond",
    "score": 0.48926767676767674,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-nano-2025-04-14'; run id azWfeFPP6zSocuQX3JYBU5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024849734950435368.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "math-level-5",
    "score": 0.6999622356495468,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-nano-2025-04-14'; run id K4TfMXsaPd4Z46zFmiV4Rf; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010231591350302844.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "otis-mock-aime",
    "score": 0.28888888888888886,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-nano-2025-04-14'; run id 3qLHwibDhgtWEbKGwxUYVa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05970019560922746.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simpleqa-verified",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4.1-nano-2025-04-14'; run id A4JEeJR5LQfLR3Hb8z7kL2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007513751157474971.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "gpqa-diamond",
    "score": 0.4236111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4-1106-preview'; run id AWewipyfSQb8hXtS8d4fNL; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024076142618301676.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "math-level-5",
    "score": 0.40020770392749244,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4-1106-preview'; run id VBStRVGFi47EVz2pn33xu4; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010538794632397387.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "gpqa-diamond",
    "score": 0.6868686868686869,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4.5-preview-2025-02-27'; run id dodibSv7pk8GtqeMy4YLQg; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.033042050878136546.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "math-level-5",
    "score": 0.7862537764350453,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-28",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4.5-preview-2025-02-27'; run id japqZFi3FS7um32EHG6wwW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011270694428136297.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "otis-mock-aime",
    "score": 0.37777777777777777,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4.5-preview-2025-02-27'; run id 9DCJMt7YsTMDenTdX4Db6V; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07309112127323451.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "epoch-chess",
    "score": 0.06,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4-turbo-2024-04-09'; run id gDyWKGy2wyoMhb2Jh3NQcE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023868325657594204.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "frontiermath",
    "score": 0.007017543859649123,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-4-turbo-2024-04-09'; run id bC2dbN4x2GWWmnnfseKxgQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004953408961963604.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "gpqa-diamond",
    "score": 0.4659090909090909,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4-turbo-2024-04-09'; run id TaCoN7eyPQEfDWXqs2MpcF; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026576883332580552.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "math-level-5",
    "score": 0.46733383685800606,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4-turbo-2024-04-09'; run id AZAJomaSEfZtqYqXUre9fS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01142825893039598.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "otis-mock-aime",
    "score": 0.06666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4-turbo-2024-04-09'; run id MZABVPEY8VCPAyCZ4iir68; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024034381938205007.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "gpqa-diamond",
    "score": 0.4889520202020202,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-05-13'; run id Y59q8WbEDp2HH8oovyJCj9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026254109873371994.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "math-level-5",
    "score": 0.5104796072507553,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-05-13'; run id Uqsvq9TAGoqD2xpKfhHBmS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01120841984218505.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "otis-mock-aime",
    "score": 0.0625,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-05-13'; run id 9cCn5aW522cTRxJSE69crm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023249986423369555.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "epoch-chess",
    "score": 0.13,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id g2FXXomRgJgYN4y2wBXBjW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03379976689896311.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "frontiermath",
    "score": 0.0035087719298245615,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id ikS95CXY2foitnUPDKwNJn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0035087719298245615.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "gpqa-diamond",
    "score": 0.49210858585858586,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id fairJpL9VekDBKxYHG9ZoG; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025999270277271707.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "math-level-5",
    "score": 0.5327605740181269,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id ZQgz8AGm4KHin9MFupLu58; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01122129021441016.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "otis-mock-aime",
    "score": 0.06388888888888888,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id mBUKqiVQsqXd3qmdtKP5UP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025630380139261548.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simpleqa-verified",
    "score": 0.26,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-08-06'; run id bDFqfWxMG7gLDq4Mz8PRTP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01387777332977422.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "gpqa-diamond",
    "score": 0.4788510101010101,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-05",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-11-20'; run id gULKGqAe9nG9syxmYMtkhi; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026336972178726107.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "math-level-5",
    "score": 0.49773413897280966,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-05",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-11-20'; run id D3G2BKGnh6YSvw4bEYSdiY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011102819618096572.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "otis-mock-aime",
    "score": 0.0625,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-11-20'; run id buNVL2TWcdVgxAcMfVE8xJ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024165360466267546.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "swe-bench-verified",
    "score": 0.30991735537190085,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-02-11",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4o-2024-11-20'; run id PncXWJm7ZCY3qv982pKoRy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021042625979133336. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id Q3eZqReiCfGFdTUSSJtFS5; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "frontiermath",
    "score": 0.007017543859649123,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id 6XhtKr59amKko65T7RUB2a; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004953408961963604.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "gpqa-diamond",
    "score": 0.37720959595959597,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id L9yfwZC3snsKZshBXTaXjc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024068925424620136.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "math-level-5",
    "score": 0.5263406344410876,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id 5iyKgWVhAmNwrA5rNL7MbW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011275275696871733.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "epoch-mystery-games",
    "score": 0.12,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id mbviVfQ7Bh7wHSCDDmrt7u; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03265986323710904.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "otis-mock-aime",
    "score": 0.06944444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-07-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id MPJkYPEMsQrowqtecK3mmz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0273963714244758.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simpleqa-verified",
    "score": 0.083,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-4o-mini-2024-07-18'; run id DLr586bZUHpghAf8m3GRSm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008728527206074758.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.46,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-pro-2025-12-11_xhigh'; run id oALBfwNc7dcNSDJSSBJXYe; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07789619230571372.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-2-pro",
    "bench": "frontiermath",
    "score": 0.74,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.2-pro-2025-12-11_xhigh'; run id dMJenf5xYWu7Pwn2U6w4dW; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026349537704451795.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "epoch-chess",
    "score": 0.586,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-03-19",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-pro-2026-03-05_xhigh'; run id gpt-5-4-pro-chess-cloud-run; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.5853658536585366,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-pro-2026-03-05_xhigh'; run id W6b3C8AeNC7CAFs5DQma8N; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07789619230571372.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "frontiermath",
    "score": 0.8245614035087719,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-13",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-pro-2026-03-05_xhigh'; run id ZFVa8dw7oiir6PhcF8XZMo; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022569134425265196.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "gpqa-diamond",
    "score": 0.946,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-03-20",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-pro-2026-03-05_xhigh'; run id gpt-5-4-pro-gpqa-cloud-run; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "simpleqa-verified",
    "score": 0.463,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'gpt-5.4-pro-2026-03-05_xhigh'; run id RfSJbuuuMkeVHQj7xrth4u; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015775927227262426.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-pro",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.1951219512195122,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-pro-2025-10-06_high'; run id HfDahF4ZXTN6bjfDFwcHTU; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06265967111543964.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-pro",
    "bench": "frontiermath",
    "score": 0.5578947368421052,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-12",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'gpt-5-pro-2025-10-06_high'; run id TFDQUeRKbWiEodRQRrKdpy; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029469977528223976.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'granite-4.0-1b'; run id kCXMdn7EZfSYojajBfEYyr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gpqa-diamond",
    "score": 0.2398989898989899,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'granite-4.0-1b'; run id oWiE4CniV7k5adL5wDh5r8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01619451031298072.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "otis-mock-aime",
    "score": 0.008333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'granite-4.0-1b'; run id PiFfhBoFbnaoqX5z2gVvra; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004700633956814717.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'granite-4.0-350m'; run id QRSncCDfDRedDyS3bH6oXq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "gpqa-diamond",
    "score": 0.11237373737373738,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'granite-4.0-350m'; run id F6xZSehzKjpybEh8JkGwni; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009083571075486867.",
    "source_id": "epoch-internal",
    "invalid": "Far below guessing. At 350M this is more plausibly a format failure (no parseable letter) than knowledge 'worse than random'."
  },
  {
    "model": "granite-4-0-350m",
    "bench": "otis-mock-aime",
    "score": 0.002777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'granite-4.0-350m'; run id aQYiADNH3qVzZkWUmQn6nZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.002777777777777778.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-2-1212",
    "bench": "gpqa-diamond",
    "score": 0.5378787878787878,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-2-1212'; run id 83DRJD4FvjwqU9eStRaNiU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027066342316607196.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-2-1212",
    "bench": "math-level-5",
    "score": 0.6351963746223565,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'grok-2-1212'; run id Kpck3pLw3veM5Mt7WZAb8G; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01323438169590836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-2-1212",
    "bench": "otis-mock-aime",
    "score": 0.11527777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-2-1212'; run id PpSVDb67Ucbxpcd8JQv5ii; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03382424700930314.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-beta",
    "bench": "gpqa-diamond",
    "score": 0.7575757575757576,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-3-beta'; run id FQ8Ykq4JWRSXnyF9rbmLc7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026863078746340353.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-beta",
    "bench": "math-level-5",
    "score": 0.8874622356495468,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'grok-3-beta'; run id i9eqMbaVTDDuCxWxzGHxyU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008688487093324447.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-beta",
    "bench": "otis-mock-aime",
    "score": 0.5555555555555556,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-3-beta'; run id DKTzh9kSoipZGkDsC4NGf3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07491109582924914.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "gpqa-diamond",
    "score": 0.7550505050505051,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-26",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'grok-3-mini-beta_high'; run id JFZMWFVGNi5R5TdZrvQUFw; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027675521820041432.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "math-level-5",
    "score": 0.8806646525679759,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'grok-3-mini-beta_high'; run id HMm8XaDBr4CQdMFddasEWb; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008912712785411623.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "otis-mock-aime",
    "score": 0.7777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'grok-3-mini-beta_high'; run id fPtize44vDPcDAzMFuAurw; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06267511942419625.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-0709",
    "bench": "epoch-chess",
    "score": 0.28,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'grok-4-0709'; run id gTh3CHLgfBDhvJCwobjju6; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04512608598542129.",
    "source_id": "epoch-internal"
  },
  {
    "model": "hermes-2-theta-llama-3-70b",
    "bench": "gpqa-diamond",
    "score": 0.3746843434343434,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Hermes-2-Theta-Llama-3-70B'; run id DHdeZNynu4VtRp2VGkCGvd; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024803009993948306.",
    "source_id": "epoch-internal"
  },
  {
    "model": "hermes-2-theta-llama-3-70b",
    "bench": "math-level-5",
    "score": 0.22686933534743203,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Hermes-2-Theta-Llama-3-70B'; run id ZAVTwsDqrvbWbqSgeNdXFd; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008127708089237323.",
    "source_id": "epoch-internal"
  },
  {
    "model": "hermes-2-theta-llama-3-70b",
    "bench": "otis-mock-aime",
    "score": 0.025,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Hermes-2-Theta-Llama-3-70B'; run id 63MnE4e4AXvbQ585MaWTFv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012309149097933271.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Llama-2-13b-chat'; run id Z6uni62AdCNgrsxETZP9uN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "gpqa-diamond",
    "score": 0.26325757575757575,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-2-70b-chat-hf'; run id evPjJw9SKqq3TieSdxrSQn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019901808558284103. Epoch label 'Llama-2-70b-chat-hf' is the Hugging Face-format release of Llama 2 70B Chat.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "math-level-5",
    "score": 0.032854984894259816,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-2-70b-chat-hf'; run id 4bkXhqCNoe6RpTe9taJ7kR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00254222977415661. Epoch label 'Llama-2-70b-chat-hf' is the Hugging Face-format release of Llama 2 70B Chat.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "otis-mock-aime",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-2-70b-chat-hf'; run id NLk2oE4rzSFihPzF95peHr; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. Epoch label 'Llama-2-70b-chat-hf' is the Hugging Face-format release of Llama 2 70B Chat.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'llama-2-7b-chat'; run id c8jPtpQkXnKFXG6hAKfsH6; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "gpqa-diamond",
    "score": 0.46275252525252525,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-Tulu-3-70B-DPO'; run id 5JgivNDZp2D7Lc7cpDhS28; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02405836362093961.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "math-level-5",
    "score": 0.426642749244713,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-Tulu-3-70B-DPO'; run id Hf2ESooSteQK99yFjhFm8t; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010405854683575872.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "otis-mock-aime",
    "score": 0.044444444444444446,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.1-Tulu-3-70B-DPO'; run id FGcofc566CradiNmHUGqVB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01827278031830863.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "gpqa-diamond",
    "score": 0.41035353535353536,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-90B-Vision-Instruct'; run id 9SSHEP95FYVZvHDdoHSRy3; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022948341091025694.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "math-level-5",
    "score": 0.39435422960725075,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-90B-Vision-Instruct'; run id iHRmwxsh8M3FGun8x5mxkt; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010293053094701802.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "otis-mock-aime",
    "score": 0.02638888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Llama-3.2-90B-Vision-Instruct'; run id LHRL6nQJZXGJmYppR7kbGw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00941990275434065.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-70b",
    "bench": "gpqa-diamond",
    "score": 0.40561868686868685,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-70B-Instruct'; run id jrvutCoFFG4KxYtU7REdpe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.026224849492112068.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-70b",
    "bench": "math-level-5",
    "score": 0.225547583081571,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-70B-Instruct'; run id dekbZbU3BtiQZNKG8Txaax; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008257639939780867.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-70b",
    "bench": "otis-mock-aime",
    "score": 0.043055555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-70B-Instruct'; run id 6RD3A6J432TZzxrKYRxph2; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02192429710598427.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-8b",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-8B-Instruct'; run id DQKgb3kZnm32FxHpQudBkf; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-8b",
    "bench": "gpqa-diamond",
    "score": 0.26073232323232326,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-8B-Instruct'; run id iT3BBFWjsZk2rKHKeJDsM8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01703043361773576.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-8b",
    "bench": "math-level-5",
    "score": 0.0612726586102719,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-8B-Instruct'; run id BURdKU7mhUX4YMPUDoFsed; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0037433164863856527.",
    "source_id": "epoch-internal"
  },
  {
    "model": "llama-3-8b",
    "bench": "otis-mock-aime",
    "score": 0.019444444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Meta-Llama-3-8B-Instruct'; run id CAcNPzYBEu39HFWPmuUEeP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010477515876628662.",
    "source_id": "epoch-internal"
  },
  {
    "model": "magistral-small-2506",
    "bench": "gpqa-diamond",
    "score": 0.5606060606060606,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'magistral-small-2506'; run id dwGLob8Vhrwx4Wn376jVmt; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0353608594752948.",
    "source_id": "epoch-internal"
  },
  {
    "model": "magistral-small-2506",
    "bench": "otis-mock-aime",
    "score": 0.3,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'magistral-small-2506'; run id hmZPoiHBpqufv9r7ujiK8V; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05924610544247621.",
    "source_id": "epoch-internal"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "gpqa-diamond",
    "score": 0.25252525252525254,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'ministral-3b-2410'; run id bJV4D8SEo9zMJPEYTTXJNZ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019305614722797293.",
    "source_id": "epoch-internal"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "math-level-5",
    "score": 0.14444864048338368,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'ministral-3b-2410'; run id YNhHxRT9Ee2uEWWJh9RXuP; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007446287617112978.",
    "source_id": "epoch-internal"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "gpqa-diamond",
    "score": 0.27146464646464646,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'ministral-8b-2410'; run id DGEzZA6nvTne4MEGSuo3Tm; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020863706671553732.",
    "source_id": "epoch-internal"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "math-level-5",
    "score": 0.14935800604229607,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'ministral-8b-2410'; run id hqdiKcvEPSRyvenUVB9qxM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007454534670584494.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Mistral-7B-Instruct-v0.3'; run id MkjPqUsNbNWjua7cP32apa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "gpqa-diamond",
    "score": 0.1518308080808081,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Mistral-7B-Instruct-v0.3'; run id YbQkMPC2GtNLhxqTEhhVsa; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010148891037227303.",
    "source_id": "epoch-internal",
    "invalid": "Below the 25% guess rate, while Epoch's Llama-3.1-8B-Instruct gets 0.27 and Llama-3-8B 0.26. Probably unparsed answers. Epoch's open-mistral-7b also scores 0.132."
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "math-level-5",
    "score": 0.03597054380664653,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Mistral-7B-Instruct-v0.3'; run id 6xHApTdQAWM9e5oCvKYS3W; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.002666943446495743.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "otis-mock-aime",
    "score": 0.002777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Mistral-7B-Instruct-v0.3'; run id j6Mo25oV9Y9jdGxCMHZfpU; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.002777777777777778.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2402",
    "bench": "gpqa-diamond",
    "score": 0.38762626262626265,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-large-2402'; run id dDeREdDJ5bVyci3APZxzWb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024776598535417284.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2402",
    "bench": "math-level-5",
    "score": 0.24461858006042297,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-large-2402'; run id gypQecfFNK2x4pNGDtGMDv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00892334786643176.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2402",
    "bench": "otis-mock-aime",
    "score": 0.019444444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-large-2402'; run id ZVHvk3GTPUNTgjc5w3vkhW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00948945749721607.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2407",
    "bench": "gpqa-diamond",
    "score": 0.49021464646464646,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-large-2407'; run id ZQPNnicTuhuscJvni85PFY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025918840988656967.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2407",
    "bench": "math-level-5",
    "score": 0.4481684290030212,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-large-2407'; run id div9uE9eovqykqx883kSpE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010703488686415494.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-2407",
    "bench": "otis-mock-aime",
    "score": 0.08472222222222223,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-large-2407'; run id bHLfPfZj8HgD9Gef75NtrM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024742642797709363.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "gpqa-diamond",
    "score": 0.5953282828282829,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-05-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-medium-2505'; run id FNMbfvEEm9PRnrJnrqk2PH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02824697229654645.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "math-level-5",
    "score": 0.8162764350453172,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-05-07",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-medium-2505'; run id LNUnvvrDra89Vhdc9bzdCy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.00899565165100951.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "otis-mock-aime",
    "score": 0.32222222222222224,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-05-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-medium-2505'; run id QWw65xeTn8x42SymMsVREN; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05965611798482264.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "gpqa-diamond",
    "score": 0.2989267676767677,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'open-mistral-nemo-2407'; run id XCSWtbDquA7FUiWfd7kh9q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02096191098761666. Epoch label is the Mistral API id open-mistral-nemo-2407 (Mistral legacy-models table: 'Mistral Nemo 12B', version 24.07).",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "math-level-5",
    "score": 0.1082892749244713,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'open-mistral-nemo-2407'; run id 9qCrggpoq8FjQWVQhHaGBT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006092598712054613. Epoch label is the Mistral API id open-mistral-nemo-2407 (Mistral legacy-models table: 'Mistral Nemo 12B', version 24.07).",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3",
    "bench": "epoch-chess",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'mistral-small-3-2501'; run id iBTrxsUHM7E4MwPnCjqETD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. Epoch labels 'mistral-small-2501' (Mistral API id, 'Mistral Small 3.0' 25.01) and 'mistral-small-3-2501' are both model_metadata group 'Mistral Small 3' and name the same 2501 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3",
    "bench": "gpqa-diamond",
    "score": 0.47285353535353536,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-small-3-2501'; run id YJPwhEJa6qQWB23HQjByxt; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027888880246470463. Epoch labels 'mistral-small-2501' (Mistral API id, 'Mistral Small 3.0' 25.01) and 'mistral-small-3-2501' are both model_metadata group 'Mistral Small 3' and name the same 2501 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3",
    "bench": "math-level-5",
    "score": 0.4481684290030212,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-30",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-small-2501'; run id NbsnvBsMoMizozbPZY8LLb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010854927233993669. Epoch labels 'mistral-small-2501' (Mistral API id, 'Mistral Small 3.0' 25.01) and 'mistral-small-3-2501' are both model_metadata group 'Mistral Small 3' and name the same 2501 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3",
    "bench": "otis-mock-aime",
    "score": 0.06666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-small-3-2501'; run id LEBmfabfasGXFwRwhN9j35; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02682716849943299. Epoch labels 'mistral-small-2501' (Mistral API id, 'Mistral Small 3.0' 25.01) and 'mistral-small-3-2501' are both model_metadata group 'Mistral Small 3' and name the same 2501 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.1-2503'; run id F7SuanDZYsPw66mJ6Vpu4C; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998. Epoch labels 'mistral-small-2503' (Mistral API id, 'Mistral Small 3.1' 25.03) and 'mistral-small-3.1-2503' are both model_metadata group 'Mistral Small 3.1' and name the same 2503 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "gpqa-diamond",
    "score": 0.41919191919191917,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.1-2503'; run id DvbGrBaybHM3UhJwWhE3LR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02348827340432996. Epoch labels 'mistral-small-2503' (Mistral API id, 'Mistral Small 3.1' 25.03) and 'mistral-small-3.1-2503' are both model_metadata group 'Mistral Small 3.1' and name the same 2503 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "math-level-5",
    "score": 0.4677114803625378,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-03-18",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'mistral-small-2503'; run id Kas7Y5PBsT7Li5Wm6Y8mj8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011995700377965737. Epoch labels 'mistral-small-2503' (Mistral API id, 'Mistral Small 3.1' 25.03) and 'mistral-small-3.1-2503' are both model_metadata group 'Mistral Small 3.1' and name the same 2503 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "otis-mock-aime",
    "score": 0.03888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'mistral-small-3.1-2503'; run id nNZHFNyHgfv3uzPAxspaPT; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01529498571765449. Epoch labels 'mistral-small-2503' (Mistral API id, 'Mistral Small 3.1' 25.03) and 'mistral-small-3.1-2503' are both model_metadata group 'Mistral Small 3.1' and name the same 2503 release; the selection rule is applied across both labels.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "gpqa-diamond",
    "score": 0.3058712121212121,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Mixtral-8x7B-Instruct-v0.1'; run id Sn2qbB2HiySe7NcG28jnyM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01937059603941351.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "math-level-5",
    "score": 0.09290030211480363,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Mixtral-8x7B-Instruct-v0.1'; run id 4PTATufH2ra3guNeLwT9p9; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005102464546317853.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "epoch-chess",
    "score": 0.15,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id oZ2GUFDoLA9sDdyefwgHPE; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.035887028128263665.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "frontiermath",
    "score": 0.14736842105263157,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id NMJhDtxtyQJCHQ4EApr9LM; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021034091168852485.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "gpqa-diamond",
    "score": 0.7676767676767676,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id NAkCFiFiDSN3NXMj7HMvMS; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030088629490217445.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "math-level-5",
    "score": 0.947129909365559,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id jVXtRkTBDnqH8p4sjVUbgN; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006152192005965586.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "otis-mock-aime",
    "score": 0.7333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-07-15",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id AEQDjKHZzrVUubU42ZUZwA; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06666666666666667.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simpleqa-verified",
    "score": 0.411,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'o1-2024-12-17_high'; run id LQfu4ngArcgMEyQZxuR7r4; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01556667341859915.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-mini",
    "bench": "gpqa-diamond",
    "score": 0.6237373737373737,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o1-mini-2024-09-12_high'; run id X2gKewiGPZx5DLFEddUUqW; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027427723511230753.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-mini",
    "bench": "math-level-5",
    "score": 0.8918051359516617,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o1-mini-2024-09-12_high'; run id 4DWnpJowAfeQDHc2xDNy2C; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006802820166065648.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-mini",
    "bench": "otis-mock-aime",
    "score": 0.46944444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o1-mini-2024-09-12_high'; run id 9zhPZLvNeUPQ7JBSpTwKPp; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06456873681038247.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-preview",
    "bench": "gpqa-diamond",
    "score": 0.5031565656565656,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o1-preview-2024-09-12'; run id 7CkADTsoNbDd88uqYbqfwj; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028257821072909393.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-preview",
    "bench": "math-level-5",
    "score": 0.8164652567975831,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o1-preview-2024-09-12'; run id 8Uu6XsSJCKwcwhXDt8vgzq; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010642608258161584.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o1-preview",
    "bench": "otis-mock-aime",
    "score": 0.3111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o1-preview-2024-09-12'; run id Cprjaj9JQ9MY5NCXqeQR5H; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0697920592732311.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "epoch-chess",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id 52bwKmNetKXPAGYaBjdgCB; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.03775251680686369.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "frontiermath-tier-4-v2",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id iR3Ut7aYEtz8dTYQFzq7rG; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "frontiermath",
    "score": 0.18596491228070175,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-11",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id mRJaJyBY2M2GGdbdMmT6iq; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023087552563757593.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "gpqa-diamond",
    "score": 0.7702020202020202,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id MzsTvCEEfEpzn4WvxYdJkX; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025855332698878485.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "math-level-5",
    "score": 0.9648791540785498,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-02-13",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id 3VDhPT5WvzjJZkRc36Kuvi; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004265263461984669.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id JSmdEXGULNBJ7PfSoVoa7W; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "otis-mock-aime",
    "score": 0.7694444444444445,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-02-27",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id Mjx6vYbgAb2DHXK8xcupX8; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05239762076828802.",
    "source_id": "epoch-internal"
  },
  {
    "model": "o3-mini",
    "bench": "simpleqa-verified",
    "score": 0.153,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-31",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'o3-mini-2025-01-31_high'; run id UCba4H3NNtUiWXMisWnh9g; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011389500459665504.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "gpqa-diamond",
    "score": 0.2758838383838384,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Phi-3-medium-128k-instruct'; run id bDS2AGEf3AywbVcdDyAKZK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016117685500292727.",
    "source_id": "epoch-internal"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "math-level-5",
    "score": 0.17560422960725075,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Phi-3-medium-128k-instruct'; run id njA86GC9DzSXyiZWtUxXiD; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.007295657661098519.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "gpqa-diamond",
    "score": 0.5612373737373737,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen-max-2025-01-25'; run id GBf7rqHkGGzfX87V37F9mH; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027858831475709448.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "math-level-5",
    "score": 0.6718277945619335,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen-max-2025-01-25'; run id GFZDNH95TnVegNrEtmUT2Q; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011143967442882196.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "otis-mock-aime",
    "score": 0.16111111111111112,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-01",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen-max-2025-01-25'; run id PZt4Yb9bpXVPDoPi4uTxwK; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04102194808229969.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-plus-2025-01-25",
    "bench": "gpqa-diamond",
    "score": 0.4810606060606061,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen-plus-2025-01-25'; run id QtraMoKiQRRpZc9txpFAYF; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027144936812277986.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-plus-2025-01-25",
    "bench": "math-level-5",
    "score": 0.6527567975830816,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen-plus-2025-01-25'; run id J3WoEpLKAJZWGGpyBVCCCn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011334654774046507.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-plus-2025-01-25",
    "bench": "otis-mock-aime",
    "score": 0.17777777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen-plus-2025-01-25'; run id AULNJG2C75JBP6uaiAgj3P; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04360797453817489.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-turbo-2024-11-01",
    "bench": "gpqa-diamond",
    "score": 0.41792929292929293,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen-turbo-2024-11-01'; run id 9Ke5At9bYuf56SmKJJ7Sz7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02617727508870251.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-turbo-2024-11-01",
    "bench": "math-level-5",
    "score": 0.5623111782477341,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen-turbo-2024-11-01'; run id jWV936hYKg6rYcPEynzEEt; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011364630947550552.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen-turbo-2024-11-01",
    "bench": "otis-mock-aime",
    "score": 0.06111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-04-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen-turbo-2024-11-01'; run id G5gxbTH33S7LXA29o5fgmv; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.022340279838327228.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "gpqa-diamond",
    "score": 0.307449494949495,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen1.5-32b-chat'; run id 9K6c9b6v9zQrQYtw827BDn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020200517973424446.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "gpqa-diamond",
    "score": 0.2881944444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen1.5-72b-chat'; run id NeqeQSxHmdUNLk9Ytpo6qo; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01993691228122493.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-72b",
    "bench": "gpqa-diamond",
    "score": 0.4078282828282828,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen2-72b-instruct'; run id Pex6SpYJX8ZPySncfNNy7H; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02574288051254005.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen2-72b",
    "bench": "math-level-5",
    "score": 0.39067220543806647,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen2-72b-instruct'; run id 9mWoY9fM2ifKvDg3xaug6h; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.010317258861464411.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "epoch-chess",
    "score": 0.04,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'Qwen3-30B-A3B'; run id JCwFm63cUipLHBfM6BdssQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019694638556693237.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.6167929292929293,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Qwen3-30B-A3B'; run id BnX67AoZzgVbHgaM8F8Brz; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029727711361608088.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "otis-mock-aime",
    "score": 0.6277777777777778,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'Qwen3-30B-A3B'; run id XfjRkSyP4ub4gDsWcgS7KY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.062118645754659016.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-4b",
    "bench": "epoch-chess",
    "score": 0.01,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-4b_none'; run id 3mCgC7byPaZ3zzPM98bY5A; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.009999999999999998.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-4b",
    "bench": "gpqa-diamond",
    "score": 0.5233585858585859,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-4b'; run id G3dinPvwrci2Nu3m9pby2i; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.030102242301033352.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "epoch-chess",
    "score": 0.21,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id dWndu92zERWxzbpNa4BdJy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.040936018074033236.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "frontiermath",
    "score": 0.09473684210526316,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id 6XZAenohp9pCgQ3gVeUAi8; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017377516319276178.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "gpqa-diamond",
    "score": 0.8232323232323232,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id S7wbot7JR53BAjM8F9gaws; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027178752639044908.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "epoch-mystery-games",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id b9csPZ2c9CAsRXmzzfVoEW; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "otis-mock-aime",
    "score": 0.8444444444444444,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id awDKyKbPXRzBVZzfdQucef; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05463890236888291.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "simpleqa-verified",
    "score": 0.2032032032032032,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.5-flash'; run id K8zTckSXLKLesoLBv24TWc; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012737194890497593.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "epoch-chess",
    "score": 0.22,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-plus'; run id UgqBt5cFjsC96ALwiTFCPR; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04163331998932266.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "gpqa-diamond",
    "score": 0.8484848484848485,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.5-plus'; run id mGXYSUNCMisvaRzhQp6g2L; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.025545650426603637.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "epoch-mystery-games",
    "score": 0.16,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.5-plus'; run id kCQr6ARwJyUfkwWqKAfuDX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.036845294917747066.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "otis-mock-aime",
    "score": 0.8666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.5-plus'; run id cFbntXuxUtfreULvt7vEt7; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051247074319053836.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "simpleqa-verified",
    "score": 0.2535070140280561,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.5-plus'; run id jrawnM2wHFbVfJFJsap4kS; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013777178222916077.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview'; run id i5cWL9DPrmhvacKyYf8gxE; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "gpqa-diamond",
    "score": 0.8737373737373737,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview'; run id Jv2hJvjpvA8xJW3Bzv7bfw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023664359402880177.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "epoch-mystery-games",
    "score": 0.19,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview_none'; run id mbySBWyYrKdtd7joxHyBdq; effort none; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.039427724440366255.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "otis-mock-aime",
    "score": 0.9111111111111111,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview'; run id EfJpx5HztMmL5EQSwsyqMe; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04290254662948546.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "simpleqa-verified",
    "score": 0.5200400801603207,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview'; run id iyPiR8GUuyNNA6Fssziq32; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015822434816352416.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "swe-bench-verified",
    "score": 0.7665289256198347,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-05-28",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-09-25; exact model version 'qwen3.6-max-preview'; run id FkDMVLdEnJDR5PyUCT6BNQ; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01924895909142592. Epoch agent scaffold, not a bare-model coding result.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "epoch-chess",
    "score": 0.04,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2025-12-10",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id Wfe5udNStR4yt4eoqgZ8hY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01969463855669324.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "frontiermath",
    "score": 0.18947368421052632,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-08-30",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id NrTH3EYHjp4hPdxgpCPGTX; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023254062737737245.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "gpqa-diamond",
    "score": 0.726010101010101,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-10-06",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id 46RCssznDADYeg8MuSgvsj; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02734528103171235.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "math-level-5",
    "score": 0.9712990936555891,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-10-09",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id f2vxirYMpXwkaZxb4ywnYM; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.004200694815744905.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "epoch-mystery-games",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id SfzmmHYZjuvWnvoYWit58g; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "otis-mock-aime",
    "score": 0.7333333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-10-06",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id ULeuPtwp3ZoyuXQYuuQ82i; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.059139451700829515.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "simpleqa-verified",
    "score": 0.4874874874874875,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-09-25; exact model version 'qwen3-max-2025-09-23'; run id fXG2GPt5F25tHRTPhAoZ4K; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01582226675546796.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwq-32b",
    "bench": "epoch-chess",
    "score": 0.05,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-09-25; exact model version 'QwQ-32B'; run id eppmTJxSfs9jxpT8N7nNTY; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021904291355759033.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwq-32b",
    "bench": "gpqa-diamond",
    "score": 0.6534090909090909,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'QwQ-32B'; run id PUXMxQiRY6XaG6NjwsEDLy; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0281399081227253.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwq-32b",
    "bench": "otis-mock-aime",
    "score": 0.5916666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-09-25; exact model version 'QwQ-32B'; run id 2xMn6dbY9mw4pmkh8Tz9Gn; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06348963965431165.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwq-plus",
    "bench": "gpqa-diamond",
    "score": 0.6540404040404041,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-04-11",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'qwq-plus'; run id PtxHF8BH8WuNdNfDnutGBb; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.028469238770829935.",
    "source_id": "epoch-internal"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "gpqa-diamond",
    "score": 0.43434343434343436,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'WizardLM-2-8x22B'; run id MQJA3RUjsQzSubGJiukZ7A; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.023232093333444247.",
    "source_id": "epoch-internal"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "math-level-5",
    "score": 0.25736404833836857,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'WizardLM-2-8x22B'; run id 4bAYQuh4heJoTNhapVZLNB; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008719521575190186.",
    "source_id": "epoch-internal"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "gpqa-diamond",
    "score": 0.319760101010101,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Yi-1.5-34B-Chat'; run id Jwi8JTeeTxJuJY8qidxngw; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.020005291661900883.",
    "source_id": "epoch-internal"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "math-level-5",
    "score": 0.2548149546827795,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Yi-1.5-34B-Chat'; run id bH5kVfLE53CxY3qWmB9C5K; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.008772368987739622.",
    "source_id": "epoch-internal"
  },
  {
    "model": "yi-34b-chat",
    "bench": "gpqa-diamond",
    "score": 0.14741161616161616,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-09-25; exact model version 'Yi-34B-Chat'; run id WxMpEBcTf8qPFPz4BrF86B; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.011372823721448395.",
    "source_id": "epoch-internal",
    "invalid": "Well below the 25% guess rate. Peers of similar age and size score 0.25 to 0.27 in the same Epoch run (deepseek-llm-67b-chat 0.246, Llama-2-70b-chat 0.263), which suggests answer-format failures."
  },
  {
    "model": "yi-34b-chat",
    "bench": "math-level-5",
    "score": 0.05145392749244713,
    "source": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv",
    "by": "independent",
    "date": "2025-01-27",
    "note": "Epoch AI internal evaluation; file math_level_5.csv; snapshot 2026-09-25; exact model version 'Yi-34B-Chat'; run id nzy4SGSeKGzQSKJLWExKpV; effort default; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0033693214361751756.",
    "source_id": "epoch-internal"
  },
  {
    "model": "sd-1-1",
    "bench": "geneval",
    "score": 0.41,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "SD v1.1; App. Table 3 (A1.T3), Overall column; reported 0.41; GenEval authors' run; same experiment as the approved README rows.",
    "evidence": {
      "reported_value": "0.41",
      "divisor": 1,
      "source_model_label": "SD v1.1",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-2",
    "bench": "geneval",
    "score": 0.44,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "SD v1.2; App. Table 3 (A1.T3), Overall column; reported 0.44; GenEval authors' run; same experiment as the approved README rows.",
    "evidence": {
      "reported_value": "0.44",
      "divisor": 1,
      "source_model_label": "SD v1.2",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-4",
    "bench": "geneval",
    "score": 0.42,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "SD v1.4; App. Table 3 (A1.T3), Overall column; reported 0.42; GenEval authors' run; same experiment as the approved README rows.",
    "evidence": {
      "reported_value": "0.42",
      "divisor": 1,
      "source_model_label": "SD v1.4",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-2-0",
    "bench": "geneval",
    "score": 0.51,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "SD v2.0; App. Table 3 (A1.T3), Overall column; reported 0.51; GenEval authors' run; same experiment as the approved README rows. default 768x768 resolution per App. C.2",
    "evidence": {
      "reported_value": "0.51",
      "divisor": 1,
      "source_model_label": "SD v2.0",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "deepfloyd-if-m",
    "bench": "geneval",
    "score": 0.52,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "IF-M; App. Table 3 (A1.T3), Overall column; reported 0.52; GenEval authors' run; same experiment as the approved README rows. stages I+II only (256x256), IF-II-M",
    "evidence": {
      "reported_value": "0.52",
      "divisor": 1,
      "source_model_label": "IF-M",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "deepfloyd-if-l",
    "bench": "geneval",
    "score": 0.54,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "IF-L; App. Table 3 (A1.T3), Overall column; reported 0.54; GenEval authors' run; same experiment as the approved README rows. stages I+II only (256x256), IF-II-L",
    "evidence": {
      "reported_value": "0.54",
      "divisor": 1,
      "source_model_label": "IF-L",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dreamlike-photoreal-2-0",
    "bench": "hpsv2-benchmark",
    "score": 0.2786,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Dreamlike Photoreal 2.0; README 'v2 benchmark' table, Averaged column; reported 27.86; per-style Animation/Concept-art/Painting/Photo = 28.24/27.60/27.59/27.99; 512x512, DPM-Solver 25 steps, CFG 7.5",
    "evidence": {
      "reported_value": "27.86",
      "divisor": 100,
      "source_model_label": "Dreamlike Photoreal 2.0",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Dreamlike Photoreal 2.0",
        "28.24",
        "27.60",
        "27.59",
        "27.99",
        "27.86"
      ]
    }
  },
  {
    "model": "sdxl-0-9",
    "bench": "hpsv2-benchmark",
    "score": 0.2773,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "SDXL Base 0.9; README 'v2 benchmark' table, Averaged column; reported 27.73; per-style Animation/Concept-art/Painting/Photo = 28.42/27.63/27.60/27.29; row added 2023-07-27; generation settings not stated",
    "evidence": {
      "reported_value": "27.73",
      "divisor": 100,
      "source_model_label": "SDXL Base 0.9",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "SDXL Base 0.9",
        "28.42",
        "27.63",
        "27.60",
        "27.29",
        "27.73"
      ]
    }
  },
  {
    "model": "openjourney",
    "bench": "hpsv2-benchmark",
    "score": 0.2745,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Openjourney; README 'v2 benchmark' table, Averaged column; reported 27.45; per-style Animation/Concept-art/Painting/Photo = 27.85/27.18/27.25/27.53; 512x512, DPM-Solver 25 steps, CFG 7.5",
    "evidence": {
      "reported_value": "27.45",
      "divisor": 100,
      "source_model_label": "Openjourney",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Openjourney",
        "27.85",
        "27.18",
        "27.25",
        "27.53",
        "27.45"
      ]
    }
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "hpsv2-benchmark",
    "score": 0.2727,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "DeepFloyd-XL; README 'v2 benchmark' table, Averaged column; reported 27.27; per-style Animation/Concept-art/Painting/Photo = 27.64/26.83/26.86/27.75; IF-I-XL + stage II (DDPM 25 steps), then resized to 192x192 and upscaled to 768x768 with the SD x4 upscaler instead of IF stage III",
    "evidence": {
      "reported_value": "27.27",
      "divisor": 100,
      "source_model_label": "DeepFloyd-XL",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "DeepFloyd-XL",
        "27.64",
        "26.83",
        "26.86",
        "27.75",
        "27.27"
      ]
    }
  },
  {
    "model": "sd-2-0",
    "bench": "hpsv2-benchmark",
    "score": 0.2717,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Stable Diffusion v2.0; README 'v2 benchmark' table, Averaged column; reported 27.17; per-style Animation/Concept-art/Painting/Photo = 27.48/26.89/26.86/27.46; 512x512, DDIM 50 steps, CFG 7.5; 768-v vs 512-base checkpoint not stated",
    "evidence": {
      "reported_value": "27.17",
      "divisor": 100,
      "source_model_label": "Stable Diffusion v2.0",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Stable Diffusion v2.0",
        "27.48",
        "26.89",
        "26.86",
        "27.46",
        "27.17"
      ]
    }
  },
  {
    "model": "sd-1-4",
    "bench": "hpsv2-benchmark",
    "score": 0.2695,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Stable Diffusion v1.4; README 'v2 benchmark' table, Averaged column; reported 26.95; per-style Animation/Concept-art/Painting/Photo = 27.26/26.61/26.66/27.27; 512x512, DDIM 50 steps, CFG 7.5",
    "evidence": {
      "reported_value": "26.95",
      "divisor": 100,
      "source_model_label": "Stable Diffusion v1.4",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Stable Diffusion v1.4",
        "27.26",
        "26.61",
        "26.66",
        "27.27",
        "26.95"
      ]
    }
  },
  {
    "model": "dall-e-2",
    "bench": "hpsv2-benchmark",
    "score": 0.2695,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "DALL·E 2; README 'v2 benchmark' table, Averaged column; reported 26.95; per-style Animation/Concept-art/Painting/Photo = 27.34/26.54/26.68/27.24; images requested from the official API (2023)",
    "evidence": {
      "reported_value": "26.95",
      "divisor": 100,
      "source_model_label": "DALL·E 2",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "DALL·E 2",
        "27.34",
        "26.54",
        "26.68",
        "27.24",
        "26.95"
      ]
    }
  },
  {
    "model": "versatile-diffusion",
    "bench": "hpsv2-benchmark",
    "score": 0.2659,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Versatile Diffusion; README 'v2 benchmark' table, Averaged column; reported 26.59; per-style Animation/Concept-art/Painting/Photo = 26.59/26.28/26.43/27.05; official model, DPM-Solver 25 steps, CFG 7.5",
    "evidence": {
      "reported_value": "26.59",
      "divisor": 100,
      "source_model_label": "Versatile Diffusion",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Versatile Diffusion",
        "26.59",
        "26.28",
        "26.43",
        "27.05",
        "26.59"
      ]
    }
  },
  {
    "model": "cogview2",
    "bench": "hpsv2-benchmark",
    "score": 0.2647,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "CogView2; README 'v2 benchmark' table, Averaged column; reported 26.47; per-style Animation/Concept-art/Painting/Photo = 26.50/26.59/26.33/26.44; 512x512 default hyper-parameters; style none/none/oil/photo per category",
    "evidence": {
      "reported_value": "26.47",
      "divisor": 100,
      "source_model_label": "CogView2",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "CogView2",
        "26.50",
        "26.59",
        "26.33",
        "26.44",
        "26.47"
      ]
    }
  },
  {
    "model": "ldm-text2im-large",
    "bench": "hpsv2-benchmark",
    "score": 0.2578,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "Latent Diffusion; README 'v2 benchmark' table, Averaged column; reported 25.78; per-style Animation/Concept-art/Painting/Photo = 25.73/25.15/25.25/26.97; official LDM model, DPM-Solver 25 steps, eta 0.3, CFG 6.0",
    "evidence": {
      "reported_value": "25.78",
      "divisor": 100,
      "source_model_label": "Latent Diffusion",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "Latent Diffusion",
        "25.73",
        "25.15",
        "25.25",
        "26.97",
        "25.78"
      ]
    }
  },
  {
    "model": "vq-diffusion-ithq",
    "bench": "hpsv2-benchmark",
    "score": 0.251,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "VQ-Diffusion; README 'v2 benchmark' table, Averaged column; reported 25.10; per-style Animation/Concept-art/Painting/Photo = 24.97/24.70/25.01/25.71; microsoft/vq-diffusion-ithq, 256x256, CFG 5.0, 100 steps",
    "evidence": {
      "reported_value": "25.10",
      "divisor": 100,
      "source_model_label": "VQ-Diffusion",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "VQ-Diffusion",
        "24.97",
        "24.70",
        "25.01",
        "25.71",
        "25.10"
      ]
    }
  },
  {
    "model": "glide-filtered",
    "bench": "hpsv2-benchmark",
    "score": 0.2355,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark",
    "source_id": "visual-hpsv2-readme",
    "by": "independent",
    "date": "2024-02-21",
    "evaluator": "HPS v2 authors (Wu et al.)",
    "source_group": "visual-hpsv2-readme:hpsv2-benchmark",
    "note": "GLIDE; README 'v2 benchmark' table, Averaged column; reported 23.55; per-style Animation/Concept-art/Painting/Photo = 23.34/23.08/23.27/24.50; official mini-glide (filtered), 256x256, CFG 3.0, 27 DDIM steps",
    "evidence": {
      "reported_value": "23.55",
      "divisor": 100,
      "source_model_label": "GLIDE",
      "table": "README 'v2 benchmark' table, Averaged column",
      "protocol": "HPD v2 benchmark prompts (Animation, Concept-art, Painting, Photo); images generated by the HPS v2 authors (settings in arXiv 2306.09341v2 App. D) and scored with HPS v2; README 'Averaged' column",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "GLIDE",
        "23.34",
        "23.08",
        "23.27",
        "24.50",
        "23.55"
      ]
    }
  },
  {
    "model": "sd-1-1",
    "bench": "tifa-v1",
    "score": 0.757,
    "source": "https://arxiv.org/html/2303.11897v3#A2.T4",
    "source_id": "visual-tifa",
    "by": "independent",
    "date": "2023-08-17",
    "evaluator": "TIFA authors (Hu et al., University of Washington)",
    "source_group": "visual-tifa:tifa-v1",
    "note": "Stable Diffusion v1.1; App. B Table 4 (A2.T4), Overall TIFA column; reported 75.7",
    "evidence": {
      "reported_value": "75.7",
      "divisor": 100,
      "source_model_label": "Stable Diffusion v1.1",
      "table": "App. B Table 4 (A2.T4), Overall TIFA column",
      "protocol": "TIFA v1.0 (4,081 prompts, 25,829 GPT-3 questions), mPLUG-large VQA; benchmark authors' own generations (Sec. 5.2, App. E)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-5",
    "bench": "tifa-v1",
    "score": 0.784,
    "source": "https://arxiv.org/html/2303.11897v3#A2.T4",
    "source_id": "visual-tifa",
    "by": "independent",
    "date": "2023-08-17",
    "evaluator": "TIFA authors (Hu et al., University of Washington)",
    "source_group": "visual-tifa:tifa-v1",
    "note": "Stable Diffusion v1.5; App. B Table 4 (A2.T4), Overall TIFA column; reported 78.4",
    "evidence": {
      "reported_value": "78.4",
      "divisor": 100,
      "source_model_label": "Stable Diffusion v1.5",
      "table": "App. B Table 4 (A2.T4), Overall TIFA column",
      "protocol": "TIFA v1.0 (4,081 prompts, 25,829 GPT-3 questions), mPLUG-large VQA; benchmark authors' own generations (Sec. 5.2, App. E)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-2-1",
    "bench": "tifa-v1",
    "score": 0.82,
    "source": "https://arxiv.org/html/2303.11897v3#A2.T4",
    "source_id": "visual-tifa",
    "by": "independent",
    "date": "2023-08-17",
    "evaluator": "TIFA authors (Hu et al., University of Washington)",
    "source_group": "visual-tifa:tifa-v1",
    "note": "Stable Diffusion v2.1; App. B Table 4 (A2.T4), Overall TIFA column; reported 82.0",
    "evidence": {
      "reported_value": "82.0",
      "divisor": 100,
      "source_model_label": "Stable Diffusion v2.1",
      "table": "App. B Table 4 (A2.T4), Overall TIFA column",
      "protocol": "TIFA v1.0 (4,081 prompts, 25,829 GPT-3 questions), mPLUG-large VQA; benchmark authors' own generations (Sec. 5.2, App. E)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "playground-v2-5",
    "bench": "genai-arena-t2i",
    "score": 1150,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "PlayGround V2.5; Table 5(a) Text-to-Image (S3.T5); reported 1150; 95% CI +21/-22; 1024x1024, 50 steps; model settings from Table 2",
    "evidence": {
      "reported_value": "1150",
      "divisor": 1,
      "source_model_label": "PlayGround V2.5",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "playground-v2",
    "bench": "genai-arena-t2i",
    "score": 1101,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "Playground V2; Table 5(a) Text-to-Image (S3.T5); reported 1101; 95% CI +14/-20; 1024x1024, 50 steps; model settings from Table 2",
    "evidence": {
      "reported_value": "1101",
      "divisor": 1,
      "source_model_label": "Playground V2",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "stable-cascade",
    "bench": "genai-arena-t2i",
    "score": 1057,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "StableCascade; Table 5(a) Text-to-Image (S3.T5); reported 1057; 95% CI +20/-24; 1024x1024, 20+10 steps; model settings from Table 2",
    "evidence": {
      "reported_value": "1057",
      "divisor": 1,
      "source_model_label": "StableCascade",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl-lightning-4-step",
    "bench": "genai-arena-t2i",
    "score": 1053,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "SDXL-Lightning; Table 5(a) Text-to-Image (S3.T5); reported 1053; 95% CI +22/-22; 1024x1024, 4 steps; model settings from Table 2",
    "evidence": {
      "reported_value": "1053",
      "divisor": 1,
      "source_model_label": "SDXL-Lightning",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "pixart-alpha",
    "bench": "genai-arena-t2i",
    "score": 1052,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "PixArt-α; Table 5(a) Text-to-Image (S3.T5); reported 1052; 95% CI +15/-19; 1024x1024, 50 steps (registry: PixArt-XL-2-1024-MS); model settings from Table 2",
    "evidence": {
      "reported_value": "1052",
      "divisor": 1,
      "source_model_label": "PixArt-α",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl",
    "bench": "genai-arena-t2i",
    "score": 1001,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "SDXL; Table 5(a) Text-to-Image (S3.T5); reported 1001; 95% CI +15/-14; 1024x1024, 50 steps (registry: stable-diffusion-xl-base-1.0); model settings from Table 2",
    "evidence": {
      "reported_value": "1001",
      "divisor": 1,
      "source_model_label": "SDXL",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl-turbo",
    "bench": "genai-arena-t2i",
    "score": 935,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "SDXL-Turbo; Table 5(a) Text-to-Image (S3.T5); reported 935; 95% CI +18/-16; 1024x1024, 1 step; model settings from Table 2",
    "evidence": {
      "reported_value": "935",
      "divisor": 1,
      "source_model_label": "SDXL-Turbo",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "openjourney",
    "bench": "genai-arena-t2i",
    "score": 853,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "OpenJourney; Table 5(a) Text-to-Image (S3.T5); reported 853; 95% CI +13/-17; 512x512, 50 steps (registry: prompthero/openjourney); model settings from Table 2",
    "evidence": {
      "reported_value": "853",
      "divisor": 1,
      "source_model_label": "OpenJourney",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "lcm-dreamshaper-v7",
    "bench": "genai-arena-t2i",
    "score": 817,
    "source": "https://arxiv.org/html/2406.04485v3#S3.T5",
    "source_id": "visual-genai-arena",
    "by": "independent",
    "date": "2024-06-04",
    "evaluator": "GenAI-Arena authors (TIGER-AI-Lab)",
    "source_group": "visual-genai-arena:genai-arena-t2i",
    "note": "LCM; Table 5(a) Text-to-Image (S3.T5); reported 817; 95% CI +20/-20; 512x512, 4 steps (registry: SimianLuo/LCM_Dreamshaper_v7); model settings from Table 2",
    "evidence": {
      "reported_value": "817",
      "divisor": 1,
      "source_model_label": "LCM",
      "table": "Table 5(a) Text-to-Image (S3.T5)",
      "protocol": "Anonymous pairwise human votes on the GenAI-Arena platform; prompts sampled from ImagenHub; Elo; leaderboard last updated 4 June 2024",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dall-e-3",
    "bench": "imagenhub-t2i-human",
    "score": 0.76,
    "source": "https://arxiv.org/html/2310.01596v4#S5.T4",
    "source_id": "visual-imagenhub",
    "by": "independent",
    "date": "2024-03-10",
    "evaluator": "ImagenHub authors (TIGER-AI-Lab)",
    "source_group": "visual-imagenhub:imagenhub-t2i-human",
    "note": "Dalle-3; Table 4 (S5.T4), Text-guided Image Generation, Overall column; reported 0.76; ± 0.08; via OpenAI API",
    "evidence": {
      "reported_value": "0.76",
      "divisor": 1,
      "source_model_label": "Dalle-3",
      "table": "Table 4 (S5.T4), Text-guided Image Generation, Overall column",
      "protocol": "Text-guided image generation; default settings, no negative prompts or prompt engineering; 3 human raters per model (Prolific + authors); Overall = average of sqrt(SC x PQ)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "imagenhub-t2i-human",
    "score": 0.59,
    "source": "https://arxiv.org/html/2310.01596v4#S5.T4",
    "source_id": "visual-imagenhub",
    "by": "independent",
    "date": "2024-03-10",
    "evaluator": "ImagenHub authors (TIGER-AI-Lab)",
    "source_group": "visual-imagenhub:imagenhub-t2i-human",
    "note": "DeepFloydIF; Table 4 (S5.T4), Text-guided Image Generation, Overall column; reported 0.59; ± 0.02; IF-I-XL + IF-II-L per the evaluator's code (4.3B in Table 3)",
    "evidence": {
      "reported_value": "0.59",
      "divisor": 1,
      "source_model_label": "DeepFloydIF",
      "table": "Table 4 (S5.T4), Text-guided Image Generation, Overall column",
      "protocol": "Text-guided image generation; default settings, no negative prompts or prompt engineering; 3 human raters per model (Prolific + authors); Overall = average of sqrt(SC x PQ)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl",
    "bench": "imagenhub-t2i-human",
    "score": 0.59,
    "source": "https://arxiv.org/html/2310.01596v4#S5.T4",
    "source_id": "visual-imagenhub",
    "by": "independent",
    "date": "2024-03-10",
    "evaluator": "ImagenHub authors (TIGER-AI-Lab)",
    "source_group": "visual-imagenhub:imagenhub-t2i-human",
    "note": "Stable Diffusion XL; Table 4 (S5.T4), Text-guided Image Generation, Overall column; reported 0.59; ± 0.03; stable-diffusion-xl-base-1.0 per the evaluator's code",
    "evidence": {
      "reported_value": "0.59",
      "divisor": 1,
      "source_model_label": "Stable Diffusion XL",
      "table": "Table 4 (S5.T4), Text-guided Image Generation, Overall column",
      "protocol": "Text-guided image generation; default settings, no negative prompts or prompt engineering; 3 human raters per model (Prolific + authors); Overall = average of sqrt(SC x PQ)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dall-e-2",
    "bench": "imagenhub-t2i-human",
    "score": 0.54,
    "source": "https://arxiv.org/html/2310.01596v4#S5.T4",
    "source_id": "visual-imagenhub",
    "by": "independent",
    "date": "2024-03-10",
    "evaluator": "ImagenHub authors (TIGER-AI-Lab)",
    "source_group": "visual-imagenhub:imagenhub-t2i-human",
    "note": "Dalle-2; Table 4 (S5.T4), Text-guided Image Generation, Overall column; reported 0.54; ± 0.04; via OpenAI API",
    "evidence": {
      "reported_value": "0.54",
      "divisor": 1,
      "source_model_label": "Dalle-2",
      "table": "Table 4 (S5.T4), Text-guided Image Generation, Overall column",
      "protocol": "Text-guided image generation; default settings, no negative prompts or prompt engineering; 3 human raters per model (Prolific + authors); Overall = average of sqrt(SC x PQ)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "openjourney",
    "bench": "imagenhub-t2i-human",
    "score": 0.5,
    "source": "https://arxiv.org/html/2310.01596v4#S5.T4",
    "source_id": "visual-imagenhub",
    "by": "independent",
    "date": "2024-03-10",
    "evaluator": "ImagenHub authors (TIGER-AI-Lab)",
    "source_group": "visual-imagenhub:imagenhub-t2i-human",
    "note": "OpenJourney; Table 4 (S5.T4), Text-guided Image Generation, Overall column; reported 0.50; ± 0.02; prompthero/openjourney (reference list)",
    "evidence": {
      "reported_value": "0.50",
      "divisor": 1,
      "source_model_label": "OpenJourney",
      "table": "Table 4 (S5.T4), Text-guided Image Generation, Overall column",
      "protocol": "Text-guided image generation; default settings, no negative prompts or prompt engineering; 3 human raters per model (Prolific + authors); Overall = average of sqrt(SC x PQ)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dall-e-3",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.6575,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "DALLE 3; App. Table 10 (S9.T10), overall score column; reported 3.63 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "3.63",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "DALLE 3",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "flux-1-1-pro",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.6175,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "FLUX 1.1; App. Table 10 (S9.T10), overall score column; reported 3.47 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4); reference: blackforestlabs Flux1.1 = FLUX1.1 [pro]",
    "evidence": {
      "reported_value": "3.47",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "FLUX 1.1",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "midjourney-v6-1",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.5825,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "Midjourney v6.1; App. Table 10 (S9.T10), overall score column; reported 3.33 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "3.33",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Midjourney v6.1",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-3-medium",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.5675,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SD 3; App. Table 10 (S9.T10), overall score column; reported 3.27 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4); listed among the open-source models; SD3 Medium was the only open SD3 checkpoint",
    "evidence": {
      "reported_value": "3.27",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SD 3",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "playground-v2-5",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.55,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "Playground v2.5; App. Table 10 (S9.T10), overall score column; reported 3.20 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "3.20",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Playground v2.5",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl-turbo",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.5375,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SDXL-Turbo; App. Table 10 (S9.T10), overall score column; reported 3.15 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "3.15",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SDXL-Turbo",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.4975,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SDXL; App. Table 10 (S9.T10), overall score column; reported 2.99 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.99",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SDXL",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "kolors",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.4825,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "Kolors; App. Table 10 (S9.T10), overall score column; reported 2.93 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.93",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Kolors",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "ssd-1b",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.4825,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SSD1B; App. Table 10 (S9.T10), overall score column; reported 2.93 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.93",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SSD1B",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.4425,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "IF; App. Table 10 (S9.T10), overall score column; reported 2.77 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4); reference: DeepFloyd IF-I-XL-v1.0",
    "evidence": {
      "reported_value": "2.77",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "IF",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "lcm-sdxl",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.4425,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "LCM-SDXL; App. Table 10 (S9.T10), overall score column; reported 2.77 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.77",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "LCM-SDXL",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "lcm-ssd-1b",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.415,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "LCM-SSD1B; App. Table 10 (S9.T10), overall score column; reported 2.66 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.66",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "LCM-SSD1B",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-2-1",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.355,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SD v2.1; App. Table 10 (S9.T10), overall score column; reported 2.42 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.42",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SD v2.1",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-5",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.3125,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SD v1.5; App. Table 10 (S9.T10), overall score column; reported 2.25 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.25",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SD v1.5",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-2",
    "bench": "evalmuse-fga-blip2-overall",
    "score": 0.3125,
    "source": "https://arxiv.org/html/2412.18150v2#S9.T10",
    "source_id": "visual-evalmuse",
    "by": "independent",
    "date": "2024-12-25",
    "evaluator": "EvalMuse-40K authors (Han et al.)",
    "source_group": "visual-evalmuse:evalmuse-fga-blip2-overall",
    "note": "SD v1.2; App. Table 10 (S9.T10), overall score column; reported 2.25 (FGA-BLIP2 predicted alignment on a 1-5 scale; stored (x-1)/4)",
    "evidence": {
      "reported_value": "2.25",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "SD v1.2",
      "table": "App. Table 10 (S9.T10), overall score column",
      "protocol": "200 representative EvalMuse prompts; images generated by each model; scored by the authors' FGA-BLIP2 alignment model on the 5-point Likert scale used for EvalMuse annotations",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dall-e-3",
    "bench": "kolorsprompts-human-overall",
    "score": 0.58,
    "source": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment",
    "source_id": "visual-kolors-readme",
    "by": "independent",
    "date": "2024-11-13",
    "evaluator": "Kuaishou Kolors team",
    "source_group": "visual-kolors-readme:kolorsprompts-human-overall",
    "note": "DALL-E 3; README Human Assessment table, Average Overall Satisfaction column; reported 3.32 (mean human overall-satisfaction rating, 1-5; stored (x-1)/4); evaluated by the Kolors team (a competitor), not the model's maker",
    "evidence": {
      "reported_value": "3.32",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "DALL-E 3",
      "table": "README Human Assessment table, Average Overall Satisfaction column",
      "protocol": "KolorsPrompts (>1,000 prompts, 14 categories); ~50 professional reviewers; 4 images per prompt, each rated 5 times on a 1-5 scale (Kolors technical report Sec. 3.3); April 2024 product versions; Kolors prompted in Chinese, others in English",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "playground-v2-5",
    "bench": "kolorsprompts-human-overall",
    "score": 0.5925,
    "source": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment",
    "source_id": "visual-kolors-readme",
    "by": "independent",
    "date": "2024-11-13",
    "evaluator": "Kuaishou Kolors team",
    "source_group": "visual-kolors-readme:kolorsprompts-human-overall",
    "note": "Playground-v2.5; README Human Assessment table, Average Overall Satisfaction column; reported 3.37 (mean human overall-satisfaction rating, 1-5; stored (x-1)/4); evaluated by the Kolors team (a competitor), not the model's maker",
    "evidence": {
      "reported_value": "3.37",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Playground-v2.5",
      "table": "README Human Assessment table, Average Overall Satisfaction column",
      "protocol": "KolorsPrompts (>1,000 prompts, 14 categories); ~50 professional reviewers; 4 images per prompt, each rated 5 times on a 1-5 scale (Kolors technical report Sec. 3.3); April 2024 product versions; Kolors prompted in Chinese, others in English",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "midjourney-v6",
    "bench": "kolorsprompts-human-overall",
    "score": 0.645,
    "source": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment",
    "source_id": "visual-kolors-readme",
    "by": "independent",
    "date": "2024-11-13",
    "evaluator": "Kuaishou Kolors team",
    "source_group": "visual-kolors-readme:kolorsprompts-human-overall",
    "note": "Midjourney-v6; README Human Assessment table, Average Overall Satisfaction column; reported 3.58 (mean human overall-satisfaction rating, 1-5; stored (x-1)/4); evaluated by the Kolors team (a competitor), not the model's maker",
    "evidence": {
      "reported_value": "3.58",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Midjourney-v6",
      "table": "README Human Assessment table, Average Overall Satisfaction column",
      "protocol": "KolorsPrompts (>1,000 prompts, 14 categories); ~50 professional reviewers; 4 images per prompt, each rated 5 times on a 1-5 scale (Kolors technical report Sec. 3.3); April 2024 product versions; Kolors prompted in Chinese, others in English",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "kolors",
    "bench": "kolorsprompts-human-overall",
    "score": 0.6475,
    "source": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment",
    "source_id": "visual-kolors-readme",
    "by": "lab",
    "date": "2024-11-13",
    "evaluator": "Kuaishou Kolors team",
    "source_group": "visual-kolors-readme:kolorsprompts-human-overall",
    "note": "Kolors; README Human Assessment table, Average Overall Satisfaction column; reported 3.59 (mean human overall-satisfaction rating, 1-5; stored (x-1)/4); Kolors team's evaluation of its own model",
    "evidence": {
      "reported_value": "3.59",
      "divisor": 4,
      "scale_offset": 1,
      "source_model_label": "Kolors",
      "table": "README Human Assessment table, Average Overall Satisfaction column",
      "protocol": "KolorsPrompts (>1,000 prompts, 14 categories); ~50 professional reviewers; 4 images per prompt, each rated 5 times on a 1-5 scale (Kolors technical report Sec. 3.3); April 2024 product versions; Kolors prompted in Chinese, others in English",
      "measurement_origin": "provider own-model evaluation",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-1-4",
    "bench": "conceptmix-k2",
    "score": 0.23,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "SD v1.4; Table 4 (S4.T4), k=2 column; reported 0.23; ±0.05; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.23",
      "divisor": 1,
      "source_model_label": "SD v1.4",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sd-2-1",
    "bench": "conceptmix-k2",
    "score": 0.29,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "SD v2.1; Table 4 (S4.T4), k=2 column; reported 0.29; ±0.05; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.29",
      "divisor": 1,
      "source_model_label": "SD v2.1",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl-turbo",
    "bench": "conceptmix-k2",
    "score": 0.35,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "SDXL Turbo; Table 4 (S4.T4), k=2 column; reported 0.35; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.35",
      "divisor": 1,
      "source_model_label": "SDXL Turbo",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "pixart-alpha",
    "bench": "conceptmix-k2",
    "score": 0.37,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "PixArt alpha; Table 4 (S4.T4), k=2 column; reported 0.37; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.37",
      "divisor": 1,
      "source_model_label": "PixArt alpha",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "conceptmix-k2",
    "score": 0.38,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "DeepFloyd IF XL v1; Table 4 (S4.T4), k=2 column; reported 0.38; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.38",
      "divisor": 1,
      "source_model_label": "DeepFloyd IF XL v1",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "sdxl",
    "bench": "conceptmix-k2",
    "score": 0.43,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "SDXL Base; Table 4 (S4.T4), k=2 column; reported 0.43; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.43",
      "divisor": 1,
      "source_model_label": "SDXL Base",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "playground-v2-5",
    "bench": "conceptmix-k2",
    "score": 0.46,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "Playground v2.5; Table 4 (S4.T4), k=2 column; reported 0.46; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.46",
      "divisor": 1,
      "source_model_label": "Playground v2.5",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "dall-e-3",
    "bench": "conceptmix-k2",
    "score": 0.61,
    "source": "https://arxiv.org/html/2408.14339v1#S4.T4",
    "source_id": "visual-conceptmix",
    "by": "independent",
    "date": "2024-08-26",
    "evaluator": "ConceptMix authors (Wu et al.)",
    "source_group": "visual-conceptmix:conceptmix-k2",
    "note": "DALL·E 3; Table 4 (S4.T4), k=2 column; reported 0.61; ±0.06; k=1..7 columns are one experiment, only k=2 imported",
    "evidence": {
      "reported_value": "0.61",
      "divisor": 1,
      "source_model_label": "DALL·E 3",
      "table": "Table 4 (S4.T4), k=2 column",
      "protocol": "300 GPT-4o-generated prompts at k=2 (3 required visual concepts); one image per prompt; GPT-4o grader; full-mark score = share of images satisfying all concepts; checkpoints in Table 8; DALL·E 3 via API defaults",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0181,
    "raw_value": 1.81,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-large-v3 F16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-large-v3.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.81% at F16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.66-1.96. Upstream openai/whisper-large-v3@06f233f converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'Q8_0': 1.82, 'Q6_K': 1.83, 'Q5_K_M': 1.84, 'Q4_K_M': 1.86}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-large-v3.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F16].err_pct",
      "raw_value": 1.81,
      "upstream_repo": "openai/whisper-large-v3",
      "upstream_commit": "06f233f"
    }
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0201,
    "raw_value": 2.01,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3-turbo.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-large-v3-turbo F16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-large-v3-turbo.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.01% at F16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.83-2.21. Upstream openai/whisper-large-v3-turbo@41f01f3 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'Q8_0': 2.01, 'Q6_K': 2.01, 'Q5_K_M': 2.03, 'Q4_K_M': 2.04}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-large-v3-turbo.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F16].err_pct",
      "raw_value": 2.01,
      "upstream_repo": "openai/whisper-large-v3-turbo",
      "upstream_commit": "41f01f3"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0168,
    "raw_value": 1.68,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-0.6b-v2.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-tdt-0.6b-v2 F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-tdt-0.6b-v2.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.68% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.54-1.83. Upstream nvidia/parakeet-tdt-0.6b-v2@1b149a3 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.68, 'Q8_0': 1.69, 'Q6_K': 1.7, 'Q5_K_M': 1.7, 'Q4_K_M': 1.72}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-tdt-0.6b-v2.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.68,
      "upstream_repo": "nvidia/parakeet-tdt-0.6b-v2",
      "upstream_commit": "1b149a3"
    }
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0195,
    "raw_value": 1.95,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-0.6b-v3.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-tdt-0.6b-v3 F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-tdt-0.6b-v3.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.95% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.79-2.12. Upstream nvidia/parakeet-tdt-0.6b-v3@6d590f7 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.95, 'Q8_0': 1.94, 'Q6_K': 1.93, 'Q5_K_M': 1.92, 'Q4_K_M': 1.98}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-tdt-0.6b-v3.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.95,
      "upstream_repo": "nvidia/parakeet-tdt-0.6b-v3",
      "upstream_commit": "6d590f7"
    }
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0145,
    "raw_value": 1.45,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-rnnt-1.1b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-rnnt-1.1b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-rnnt-1.1b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.45% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.31-1.59. Upstream nvidia/parakeet-rnnt-1.1b@a07b19e converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.45, 'Q8_0': 1.46, 'Q6_K': 1.43, 'Q5_K_M': 1.43, 'Q4_K_M': 1.41}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-rnnt-1.1b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.45,
      "upstream_repo": "nvidia/parakeet-rnnt-1.1b",
      "upstream_commit": "a07b19e"
    }
  },
  {
    "model": "canary-1b-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0192,
    "raw_value": 1.92,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b-v2.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp canary-1b-v2 F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/canary-1b-v2.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.92% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.78-2.08. Upstream nvidia/canary-1b-v2@87bc526 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.92, 'Q8_0': 1.91, 'Q6_K': 1.94, 'Q5_K_M': 1.93, 'Q4_K_M': 1.91}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/canary-1b-v2.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.92,
      "upstream_repo": "nvidia/canary-1b-v2",
      "upstream_commit": "87bc526"
    }
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0163,
    "raw_value": 1.63,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-qwen-2.5b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp canary-qwen-2.5b BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/canary-qwen-2.5b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.63% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.49-1.78. Upstream nvidia/canary-qwen-2.5b@b1469e1 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.63, 'Q8_0': 1.63, 'Q6_K': 1.63, 'Q5_K_M': 1.63, 'Q4_K_M': 1.63}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/canary-qwen-2.5b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.63,
      "upstream_repo": "nvidia/canary-qwen-2.5b",
      "upstream_commit": "b1469e1"
    }
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0126,
    "raw_value": 1.26,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/cohere-transcribe-03-2026.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp cohere-transcribe-03-2026 BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/cohere-transcribe-03-2026.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.26% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream CohereLabs/cohere-transcribe-03-2026@76b8b23 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 8. Quantized rows (not used): {'Q6_K': 1.27, 'Q5_K_M': 1.25, 'Q4_K_M': 1.24, 'F16': 1.26, 'Q8_0': 1.27}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/cohere-transcribe-03-2026.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.26,
      "upstream_repo": "CohereLabs/cohere-transcribe-03-2026",
      "upstream_commit": "76b8b23"
    }
  },
  {
    "model": "granite-speech-4-1-2b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0131,
    "raw_value": 1.31,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-speech-4.1-2b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp granite-speech-4.1-2b BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.31% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.18-1.43. Upstream ibm-granite/granite-speech-4.1-2b@8f4bb5f converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.32, 'Q8_0': 1.32, 'Q6_K': 1.29, 'Q5_K_M': 1.33, 'Q4_K_M': 1.37}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/granite-speech-4.1-2b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.31,
      "upstream_repo": "ibm-granite/granite-speech-4.1-2b",
      "upstream_commit": "8f4bb5f"
    }
  },
  {
    "model": "granite-speech-5-0-470m-turboctc",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0134,
    "raw_value": 1.34,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-speech-5.0-470m-turboctc.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp granite-speech-5.0-470m-turboctc BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/granite-speech-5.0-470m-turboctc.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.34% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.21-1.47. Upstream ibm-granite/granite-speech-5.0-470m-turboctc@18ca3c1 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 8. Quantized rows (not used): {'F16': 1.33, 'Q8_0': 1.33, 'Q6_K': 1.33, 'Q5_K_M': 1.34, 'Q4_K_M': 1.35}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/granite-speech-5.0-470m-turboctc.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.34,
      "upstream_repo": "ibm-granite/granite-speech-5.0-470m-turboctc",
      "upstream_commit": "18ca3c1"
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0212,
    "raw_value": 2.12,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-0.6b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp qwen3-asr-0.6b BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/qwen3-asr-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.12% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.96-2.29. Upstream Qwen/Qwen3-ASR-0.6B@5eb1441 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.12, 'Q8_0': 2.11, 'Q6_K': 2.11, 'Q5_K_M': 2.21, 'Q4_K_M': 2.26}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/qwen3-asr-0.6b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 2.12,
      "upstream_repo": "Qwen/Qwen3-ASR-0.6B",
      "upstream_commit": "5eb1441"
    }
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0162,
    "raw_value": 1.62,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-1.7b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp qwen3-asr-1.7b BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/qwen3-asr-1.7b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.62% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.47-1.76. Upstream Qwen/Qwen3-ASR-1.7B@7278e1e converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.62, 'Q8_0': 1.62, 'Q6_K': 1.65, 'Q5_K_M': 1.65, 'Q4_K_M': 1.81}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/qwen3-asr-1.7b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.62,
      "upstream_repo": "Qwen/Qwen3-ASR-1.7B",
      "upstream_commit": "7278e1e"
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0188,
    "raw_value": 1.88,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-mini-3b-2507.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp voxtral-mini-3b-2507 BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/voxtral-mini-3b-2507.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.88% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream mistralai/Voxtral-Mini-3B-2507@3060fe3 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 8. Quantized rows (not used): {'F16': 1.89, 'Q8_0': 1.87, 'Q6_K': 1.87, 'Q5_K_M': 1.91, 'Q4_K_M': 1.94}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/voxtral-mini-3b-2507.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.88,
      "upstream_repo": "mistralai/Voxtral-Mini-3B-2507",
      "upstream_commit": "3060fe3"
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0156,
    "raw_value": 1.56,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-small-24b-2507.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp voxtral-small-24b-2507 BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/voxtral-small-24b-2507.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.56% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream mistralai/Voxtral-Small-24B-2507@da5b424 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 8. Quantized rows (not used): {'F16': 1.57, 'Q8_0': 1.56, 'Q6_K': 1.58, 'Q5_K_M': 1.6, 'Q4_K_M': 2.11}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/voxtral-small-24b-2507.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.56,
      "upstream_repo": "mistralai/Voxtral-Small-24B-2507",
      "upstream_commit": "da5b424"
    }
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0208,
    "raw_value": 2.08,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-mini-4b-realtime-2602.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp voxtral-mini-4b-realtime-2602 BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/voxtral-mini-4b-realtime-2602.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.08% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream mistralai/Voxtral-Mini-4B-Realtime-2602@2769294 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 8. Quantized rows (not used): {'F16': 2.09, 'Q8_0': 2.07, 'Q6_K': 2.08, 'Q5_K_M': 2.08, 'Q4_K_M': 2.08}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/voxtral-mini-4b-realtime-2602.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 2.08,
      "upstream_repo": "mistralai/Voxtral-Mini-4B-Realtime-2602",
      "upstream_commit": "2769294"
    }
  },
  {
    "model": "whisper-tiny",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0749,
    "raw_value": 7.49,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-tiny.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-tiny F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-tiny.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 7.49% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 7.16-7.87. Upstream openai/whisper-tiny@169d4a4 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 7.48, 'Q8_0': 7.52, 'Q6_K': 7.54, 'Q5_K_M': 7.82, 'Q4_K_M': 7.78}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-tiny.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 7.49,
      "upstream_repo": "openai/whisper-tiny",
      "upstream_commit": "169d4a4"
    }
  },
  {
    "model": "whisper-tiny-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0577,
    "raw_value": 5.77,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-tiny.en.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-tiny.en F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-tiny.en.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 5.77% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 5.46-6.09. Upstream openai/whisper-tiny.en@87c7102 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 5.78, 'Q8_0': 5.72, 'Q6_K': 5.83, 'Q5_K_M': 5.91, 'Q4_K_M': 5.96}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-tiny.en.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 5.77,
      "upstream_repo": "openai/whisper-tiny.en",
      "upstream_commit": "87c7102"
    }
  },
  {
    "model": "whisper-base",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0511,
    "raw_value": 5.11,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-base.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-base F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-base.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 5.11% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 4.83-5.39. Upstream openai/whisper-base@e37978b converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 5.1, 'Q8_0': 5.12, 'Q6_K': 5.11, 'Q5_K_M': 5.19, 'Q4_K_M': 5.36}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-base.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 5.11,
      "upstream_repo": "openai/whisper-base",
      "upstream_commit": "e37978b"
    }
  },
  {
    "model": "whisper-base-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.043,
    "raw_value": 4.3,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-base.en.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-base.en F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-base.en.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 4.3% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 3.95-4.74. Upstream openai/whisper-base.en@911407f converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 4.13, 'Q8_0': 4.16, 'Q6_K': 4.15, 'Q5_K_M': 4.16, 'Q4_K_M': 4.29}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-base.en.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 4.3,
      "upstream_repo": "openai/whisper-base.en",
      "upstream_commit": "911407f"
    }
  },
  {
    "model": "whisper-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0334,
    "raw_value": 3.34,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-small.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-small F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-small.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 3.34% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 3.11-3.58. Upstream openai/whisper-small@973afd2 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 3.33, 'Q8_0': 3.33, 'Q6_K': 3.33, 'Q5_K_M': 3.37, 'Q4_K_M': 3.4}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-small.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 3.34,
      "upstream_repo": "openai/whisper-small",
      "upstream_commit": "973afd2"
    }
  },
  {
    "model": "whisper-small-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0311,
    "raw_value": 3.11,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-small.en.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-small.en F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-small.en.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 3.11% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.83-3.46. Upstream openai/whisper-small.en@e872752 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.97, 'Q8_0': 3.09, 'Q6_K': 2.97, 'Q5_K_M': 3.11, 'Q4_K_M': 3.09}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-small.en.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 3.11,
      "upstream_repo": "openai/whisper-small.en",
      "upstream_commit": "e872752"
    }
  },
  {
    "model": "whisper-medium",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0264,
    "raw_value": 2.64,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-medium.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-medium F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-medium.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.64% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.41-2.91. Upstream openai/whisper-medium@abdf7c3 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.63, 'Q8_0': 2.64, 'Q6_K': 2.59, 'Q5_K_M': 2.62, 'Q4_K_M': 2.59}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-medium.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.64,
      "upstream_repo": "openai/whisper-medium",
      "upstream_commit": "abdf7c3"
    }
  },
  {
    "model": "whisper-medium-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0274,
    "raw_value": 2.74,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-medium.en.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-medium.en F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-medium.en.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.74% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.52-2.99. Upstream openai/whisper-medium.en@2e98eb6 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.73, 'Q8_0': 2.72, 'Q6_K': 2.82, 'Q5_K_M': 2.75, 'Q4_K_M': 2.91}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-medium.en.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.74,
      "upstream_repo": "openai/whisper-medium.en",
      "upstream_commit": "2e98eb6"
    }
  },
  {
    "model": "whisper-large",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0272,
    "raw_value": 2.72,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-large F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-large.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.72% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.42-3.06. Upstream openai/whisper-large@4ef9b41 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.72, 'Q8_0': 2.71, 'Q6_K': 2.62, 'Q5_K_M': 2.84, 'Q4_K_M': 2.67}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-large.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.72,
      "upstream_repo": "openai/whisper-large",
      "upstream_commit": "4ef9b41"
    }
  },
  {
    "model": "whisper-large-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0267,
    "raw_value": 2.67,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v2.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp whisper-large-v2 F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/whisper-large-v2.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.67% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.23-3.33. Upstream openai/whisper-large-v2@ae46427 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.68, 'Q8_0': 2.97, 'Q6_K': 2.83, 'Q5_K_M': 2.71, 'Q4_K_M': 2.46}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/whisper-large-v2.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.67,
      "upstream_repo": "openai/whisper-large-v2",
      "upstream_commit": "ae46427"
    }
  },
  {
    "model": "canary-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0155,
    "raw_value": 1.55,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp canary-1b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/canary-1b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.55% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.41-1.69. Upstream nvidia/canary-1b@1698acf converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.55, 'Q8_0': 1.55, 'Q6_K': 1.57, 'Q5_K_M': 1.57, 'Q4_K_M': 1.55}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/canary-1b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.55,
      "upstream_repo": "nvidia/canary-1b",
      "upstream_commit": "1698acf"
    }
  },
  {
    "model": "canary-1b-flash",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0162,
    "raw_value": 1.62,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b-flash.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp canary-1b-flash F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/canary-1b-flash.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.62% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.41-1.86. Upstream nvidia/canary-1b-flash@a9a55e0 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.62, 'Q8_0': 1.62, 'Q6_K': 1.65, 'Q5_K_M': 1.64, 'Q4_K_M': 1.59}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/canary-1b-flash.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.62,
      "upstream_repo": "nvidia/canary-1b-flash",
      "upstream_commit": "a9a55e0"
    }
  },
  {
    "model": "canary-180m-flash",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0194,
    "raw_value": 1.94,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-180m-flash.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp canary-180m-flash F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/canary-180m-flash.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.94% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.78-2.11. Upstream nvidia/canary-180m-flash@b12ab41 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.94, 'Q8_0': 1.93, 'Q6_K': 1.93, 'Q5_K_M': 1.9, 'Q4_K_M': 1.93}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/canary-180m-flash.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.94,
      "upstream_repo": "nvidia/canary-180m-flash",
      "upstream_commit": "b12ab41"
    }
  },
  {
    "model": "parakeet-ctc-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0187,
    "raw_value": 1.87,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-ctc-0.6b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-ctc-0.6b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-ctc-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.87% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.71-2.03. Upstream nvidia/parakeet-ctc-0.6b@ad09ba1 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.87, 'Q8_0': 1.87, 'Q6_K': 1.84, 'Q5_K_M': 1.87, 'Q4_K_M': 1.9}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-ctc-0.6b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.87,
      "upstream_repo": "nvidia/parakeet-ctc-0.6b",
      "upstream_commit": "ad09ba1"
    }
  },
  {
    "model": "parakeet-ctc-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0185,
    "raw_value": 1.85,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-ctc-1.1b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-ctc-1.1b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-ctc-1.1b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.85% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.68-2.01. Upstream nvidia/parakeet-ctc-1.1b@a707e81 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.85, 'Q8_0': 1.85, 'Q6_K': 1.85, 'Q5_K_M': 1.84, 'Q4_K_M': 1.9}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-ctc-1.1b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.85,
      "upstream_repo": "nvidia/parakeet-ctc-1.1b",
      "upstream_commit": "a707e81"
    }
  },
  {
    "model": "parakeet-rnnt-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0162,
    "raw_value": 1.62,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-rnnt-0.6b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-rnnt-0.6b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-rnnt-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.62% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.48-1.77. Upstream nvidia/parakeet-rnnt-0.6b@c0c1f09 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.62, 'Q8_0': 1.62, 'Q6_K': 1.62, 'Q5_K_M': 1.62, 'Q4_K_M': 1.66}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-rnnt-0.6b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.62,
      "upstream_repo": "nvidia/parakeet-rnnt-0.6b",
      "upstream_commit": "c0c1f09"
    }
  },
  {
    "model": "parakeet-tdt-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0139,
    "raw_value": 1.39,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-1.1b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-tdt-1.1b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-tdt-1.1b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.39% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.24-1.52. Upstream nvidia/parakeet-tdt-1.1b@53276c6 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.39, 'Q8_0': 1.38, 'Q6_K': 1.4, 'Q5_K_M': 1.39, 'Q4_K_M': 1.42}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-tdt-1.1b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 1.39,
      "upstream_repo": "nvidia/parakeet-tdt-1.1b",
      "upstream_commit": "53276c6"
    }
  },
  {
    "model": "parakeet-tdt-ctc-110m",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0243,
    "raw_value": 2.43,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt_ctc-110m.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp parakeet-tdt_ctc-110m F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/parakeet-tdt_ctc-110m.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.43% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.25-2.6. Upstream nvidia/parakeet-tdt_ctc-110m@431a349 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 2.43, 'Q8_0': 2.43, 'Q6_K': 2.44, 'Q5_K_M': 2.47, 'Q4_K_M': 2.53}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/parakeet-tdt_ctc-110m.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.43,
      "upstream_repo": "nvidia/parakeet-tdt_ctc-110m",
      "upstream_commit": "431a349"
    }
  },
  {
    "model": "granite-4-0-1b-speech",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0142,
    "raw_value": 1.42,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-4.0-1b-speech.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp granite-4.0-1b-speech BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/granite-4.0-1b-speech.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.42% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.28-1.57. Upstream ibm-granite/granite-4.0-1b-speech@bd87ab8 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 1.42, 'Q8_0': 1.44, 'Q6_K': 1.41, 'Q5_K_M': 1.42, 'Q4_K_M': 1.48}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/granite-4.0-1b-speech.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.42,
      "upstream_repo": "ibm-granite/granite-4.0-1b-speech",
      "upstream_commit": "bd87ab8"
    }
  },
  {
    "model": "moonshine-tiny",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0458,
    "raw_value": 4.58,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-tiny.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp moonshine-tiny F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/moonshine-tiny.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 4.58% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 4.3-4.85. Upstream UsefulSensors/moonshine-tiny@390624e converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 4.58, 'Q8_0': 4.6}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/moonshine-tiny.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 4.58,
      "upstream_repo": "UsefulSensors/moonshine-tiny",
      "upstream_commit": "390624e"
    }
  },
  {
    "model": "moonshine-base",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0328,
    "raw_value": 3.28,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-base.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp moonshine-base F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/moonshine-base.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 3.28% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 3.07-3.5. Upstream UsefulSensors/moonshine-base@7a73d8d converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 3.28, 'Q8_0': 3.26}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/moonshine-base.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 3.28,
      "upstream_repo": "UsefulSensors/moonshine-base",
      "upstream_commit": "7a73d8d"
    }
  },
  {
    "model": "moonshine-streaming-tiny",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0453,
    "raw_value": 4.53,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-tiny.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp moonshine-streaming-tiny F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/moonshine-streaming-tiny.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 4.53% at F32 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream UsefulSensors/moonshine-streaming-tiny@f8e9dfd converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size None. Quantized rows (not used): {'F16': 4.53, 'Q8_0': 4.52}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/moonshine-streaming-tiny.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 4.53,
      "upstream_repo": "UsefulSensors/moonshine-streaming-tiny",
      "upstream_commit": "f8e9dfd"
    }
  },
  {
    "model": "moonshine-streaming-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0253,
    "raw_value": 2.53,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-small.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp moonshine-streaming-small F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/moonshine-streaming-small.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.53% at F32 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream UsefulSensors/moonshine-streaming-small@2c03650 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size None. Quantized rows (not used): {'F16': 2.53, 'Q8_0': 2.54}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/moonshine-streaming-small.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.53,
      "upstream_repo": "UsefulSensors/moonshine-streaming-small",
      "upstream_commit": "2c03650"
    }
  },
  {
    "model": "moonshine-streaming-medium",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0216,
    "raw_value": 2.16,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-medium.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp moonshine-streaming-medium F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/moonshine-streaming-medium.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 2.16% at F32 (the full-precision tier; F32 > native BF16 > F16 preference). Upstream UsefulSensors/moonshine-streaming-medium@57b8436 converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size None. Quantized rows (not used): {'F16': 2.16, 'Q8_0': 2.16}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/moonshine-streaming-medium.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 2.16,
      "upstream_repo": "UsefulSensors/moonshine-streaming-medium",
      "upstream_commit": "57b8436"
    }
  },
  {
    "model": "canary-1b",
    "bench": "tedlium-wer",
    "score": 0.102,
    "raw_value": 10.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "cnry-1b",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'cnry-1b', column 'clean (WER)': 10.2% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / cnry-1b",
      "column_label": "clean (WER)",
      "raw_value": "10.2"
    }
  },
  {
    "model": "whisper-medium",
    "bench": "tedlium-wer",
    "score": 0.12,
    "raw_value": 12,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-md",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-md', column 'clean (WER)': 12.0% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-md",
      "column_label": "clean (WER)",
      "raw_value": "12.0"
    }
  },
  {
    "model": "whisper-large-v2",
    "bench": "tedlium-wer",
    "score": 0.121,
    "raw_value": 12.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-lg",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-lg', column 'clean (WER)': 12.1% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-lg",
      "column_label": "clean (WER)",
      "raw_value": "12.1"
    }
  },
  {
    "model": "whisper-small",
    "bench": "tedlium-wer",
    "score": 0.123,
    "raw_value": 12.3,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-sm",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-sm', column 'clean (WER)': 12.3% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-sm",
      "column_label": "clean (WER)",
      "raw_value": "12.3"
    }
  },
  {
    "model": "whisper-base",
    "bench": "tedlium-wer",
    "score": 0.133,
    "raw_value": 13.3,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-bs",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-bs', column 'clean (WER)': 13.3% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-bs",
      "column_label": "clean (WER)",
      "raw_value": "13.3"
    }
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "tedlium-wer",
    "score": 0.135,
    "raw_value": 13.5,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "w2v2-lg-slf",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'w2v2-lg-slf', column 'clean (WER)': 13.5% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / w2v2-lg-slf",
      "column_label": "clean (WER)",
      "raw_value": "13.5"
    }
  },
  {
    "model": "whisper-tiny-en",
    "bench": "tedlium-wer",
    "score": 0.137,
    "raw_value": 13.7,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-tn.en",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-tn.en', column 'clean (WER)': 13.7% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-tn.en",
      "column_label": "clean (WER)",
      "raw_value": "13.7"
    }
  },
  {
    "model": "whisper-tiny",
    "bench": "tedlium-wer",
    "score": 0.144,
    "raw_value": 14.4,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "wsp-tn",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'wsp-tn', column 'clean (WER)': 14.4% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / wsp-tn",
      "column_label": "clean (WER)",
      "raw_value": "14.4"
    }
  },
  {
    "model": "hubert-large-ls960-ft",
    "bench": "tedlium-wer",
    "score": 0.147,
    "raw_value": 14.7,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "hubt-lg",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'hubt-lg', column 'clean (WER)': 14.7% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / hubt-lg",
      "column_label": "clean (WER)",
      "raw_value": "14.7"
    }
  },
  {
    "model": "hubert-xlarge-ls960-ft",
    "bench": "tedlium-wer",
    "score": 0.147,
    "raw_value": 14.7,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "hubt-xl",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'hubt-xl', column 'clean (WER)': 14.7% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / hubt-xl",
      "column_label": "clean (WER)",
      "raw_value": "14.7"
    }
  },
  {
    "model": "wav2vec2-large-robust-ft-libri-960h",
    "bench": "tedlium-wer",
    "score": 0.152,
    "raw_value": 15.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "w2v2-lg-rob",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'w2v2-lg-rob', column 'clean (WER)': 15.2% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / w2v2-lg-rob",
      "column_label": "clean (WER)",
      "raw_value": "15.2"
    }
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "tedlium-wer",
    "score": 0.165,
    "raw_value": 16.5,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "w2v2-lg",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'w2v2-lg', column 'clean (WER)': 16.5% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / w2v2-lg",
      "column_label": "clean (WER)",
      "raw_value": "16.5"
    }
  },
  {
    "model": "wav2vec2-base-960h",
    "bench": "tedlium-wer",
    "score": 0.188,
    "raw_value": 18.8,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "w2v2-bs",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'w2v2-bs', column 'clean (WER)': 18.8% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / w2v2-bs",
      "column_label": "clean (WER)",
      "raw_value": "18.8"
    }
  },
  {
    "model": "mms-1b-fl102",
    "bench": "tedlium-wer",
    "score": 0.302,
    "raw_value": 30.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.07937v3",
    "source_id": "speech-stt-srb-report",
    "by": "independent",
    "date": "2024-12-09",
    "date_basis": "arXiv v3 (9 Dec 2024); run date not stated",
    "engine_label": "mms-1b",
    "data_license": "CC BY 4.0",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'TEDLIUM', row 'mms-1b', column 'clean (WER)': 30.2% on TED-LIUM 3 test (1,155 utt.). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations, not error rates.",
    "evidence": {
      "kind": "table",
      "table_id": "Table 7",
      "row_label": "TEDLIUM / mms-1b",
      "column_label": "clean (WER)",
      "raw_value": "30.2"
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "picovoice-en4-batch-avg-wer",
    "score": 0.057,
    "raw_value": 5.7,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19); the tiny/base/small/medium rows are unchanged since at least 7bcec2a (2025-09-16) except tiny TED-LIUM 6.5->6.6; the large-v3 row was relabelled from 'Whisper Large (Multilingual)' with identical values",
    "engine_label": "Whisper Large V3",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 3.7,
      "librispeech_test_other_pct": 5.4,
      "tedlium_pct": 4.6,
      "commonvoice_pct": 9
    },
    "note": "Picovoice README 'English > Batch Engines Word Error Rate', row 'Whisper Large V3', column 'Average' = 5.7% (mean of LibriSpeech test-clean 3.7%, test-other 5.4%, TED-LIUM 4.6%, CommonVoice 9.0%). Identity: engine.py loads openai-whisper 'large-v3' for English (whisper.load_model(..., device='cpu')). Picovoice's own normalizer (normalizer.py). Picovoice sells Leopard/Cheetah; only the Whisper rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Batch Engines Word Error Rate",
      "row_label": "Whisper Large V3",
      "column_label": "Average",
      "raw_value": "5.7%"
    }
  },
  {
    "model": "whisper-medium-en",
    "bench": "picovoice-en4-batch-avg-wer",
    "score": 0.061,
    "raw_value": 6.1,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19); the tiny/base/small/medium rows are unchanged since at least 7bcec2a (2025-09-16) except tiny TED-LIUM 6.5->6.6; the large-v3 row was relabelled from 'Whisper Large (Multilingual)' with identical values",
    "engine_label": "Whisper Medium",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 3.3,
      "librispeech_test_other_pct": 6.2,
      "tedlium_pct": 4.6,
      "commonvoice_pct": 10.2
    },
    "note": "Picovoice README 'English > Batch Engines Word Error Rate', row 'Whisper Medium', column 'Average' = 6.1% (mean of LibriSpeech test-clean 3.3%, test-other 6.2%, TED-LIUM 4.6%, CommonVoice 10.2%). Identity: engine.py loads openai-whisper 'medium.en' for English (whisper.load_model(..., device='cpu')). Picovoice's own normalizer (normalizer.py). Picovoice sells Leopard/Cheetah; only the Whisper rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Batch Engines Word Error Rate",
      "row_label": "Whisper Medium",
      "column_label": "Average",
      "raw_value": "6.1%"
    }
  },
  {
    "model": "whisper-small-en",
    "bench": "picovoice-en4-batch-avg-wer",
    "score": 0.07,
    "raw_value": 7,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19); the tiny/base/small/medium rows are unchanged since at least 7bcec2a (2025-09-16) except tiny TED-LIUM 6.5->6.6; the large-v3 row was relabelled from 'Whisper Large (Multilingual)' with identical values",
    "engine_label": "Whisper Small",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 3.3,
      "librispeech_test_other_pct": 7.2,
      "tedlium_pct": 4.8,
      "commonvoice_pct": 12.7
    },
    "note": "Picovoice README 'English > Batch Engines Word Error Rate', row 'Whisper Small', column 'Average' = 7.0% (mean of LibriSpeech test-clean 3.3%, test-other 7.2%, TED-LIUM 4.8%, CommonVoice 12.7%). Identity: engine.py loads openai-whisper 'small.en' for English (whisper.load_model(..., device='cpu')). Picovoice's own normalizer (normalizer.py). Picovoice sells Leopard/Cheetah; only the Whisper rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Batch Engines Word Error Rate",
      "row_label": "Whisper Small",
      "column_label": "Average",
      "raw_value": "7.0%"
    }
  },
  {
    "model": "whisper-base-en",
    "bench": "picovoice-en4-batch-avg-wer",
    "score": 0.095,
    "raw_value": 9.5,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19); the tiny/base/small/medium rows are unchanged since at least 7bcec2a (2025-09-16) except tiny TED-LIUM 6.5->6.6; the large-v3 row was relabelled from 'Whisper Large (Multilingual)' with identical values",
    "engine_label": "Whisper Base",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 4.3,
      "librispeech_test_other_pct": 10.4,
      "tedlium_pct": 5.4,
      "commonvoice_pct": 17.9
    },
    "note": "Picovoice README 'English > Batch Engines Word Error Rate', row 'Whisper Base', column 'Average' = 9.5% (mean of LibriSpeech test-clean 4.3%, test-other 10.4%, TED-LIUM 5.4%, CommonVoice 17.9%). Identity: engine.py loads openai-whisper 'base.en' for English (whisper.load_model(..., device='cpu')). Picovoice's own normalizer (normalizer.py). Picovoice sells Leopard/Cheetah; only the Whisper rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Batch Engines Word Error Rate",
      "row_label": "Whisper Base",
      "column_label": "Average",
      "raw_value": "9.5%"
    }
  },
  {
    "model": "whisper-tiny-en",
    "bench": "picovoice-en4-batch-avg-wer",
    "score": 0.127,
    "raw_value": 12.7,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19); the tiny/base/small/medium rows are unchanged since at least 7bcec2a (2025-09-16) except tiny TED-LIUM 6.5->6.6; the large-v3 row was relabelled from 'Whisper Large (Multilingual)' with identical values",
    "engine_label": "Whisper Tiny",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 5.9,
      "librispeech_test_other_pct": 13.8,
      "tedlium_pct": 6.6,
      "commonvoice_pct": 24.4
    },
    "note": "Picovoice README 'English > Batch Engines Word Error Rate', row 'Whisper Tiny', column 'Average' = 12.7% (mean of LibriSpeech test-clean 5.9%, test-other 13.8%, TED-LIUM 6.6%, CommonVoice 24.4%). Identity: engine.py loads openai-whisper 'tiny.en' for English (whisper.load_model(..., device='cpu')). Picovoice's own normalizer (normalizer.py). Picovoice sells Leopard/Cheetah; only the Whisper rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Batch Engines Word Error Rate",
      "row_label": "Whisper Tiny",
      "column_label": "Average",
      "raw_value": "12.7%"
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0184,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form LS-C column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "LS-C",
      "raw_value": "1.84",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0366,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form LS-O column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "LS-O",
      "raw_value": "3.66",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeech-wer",
    "score": 0.116,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form GS column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "GS",
      "raw_value": "11.60",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "voxpopuli-en-wer",
    "score": 0.0958,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form VP column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "VP",
      "raw_value": "9.58",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "switchboard-wer",
    "score": 0.1314,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form SB column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "SB",
      "raw_value": "13.14",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "chime4-wer",
    "score": 0.1088,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form C-4 column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "C-4",
      "raw_value": "10.88",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "spgispeech-wer",
    "score": 0.0315,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (short-form SPGI column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "SPGI",
      "raw_value": "3.15",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "earnings21-10m-wer",
    "score": 0.0988,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (long-form E21 10m column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "E21 10m",
      "raw_value": "9.88",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "earnings22-10m-wer",
    "score": 0.1307,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'Whisper large-v3' in Table 3 (long-form E22 10m column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "Whisper large-v3",
      "column_label": "E22 10m",
      "raw_value": "13.07",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0192,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form LS-C column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "LS-C",
      "raw_value": "1.92",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "librispeech-test-other-wer",
    "score": 0.047,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form LS-O column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "LS-O",
      "raw_value": "4.70",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "gigaspeech-wer",
    "score": 0.148,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form GS column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "GS",
      "raw_value": "14.80",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "voxpopuli-en-wer",
    "score": 0.0734,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form VP column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "VP",
      "raw_value": "7.34",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "switchboard-wer",
    "score": 0.1731,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form SB column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "SB",
      "raw_value": "17.31",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "chime4-wer",
    "score": 0.1135,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form C-4 column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "C-4",
      "raw_value": "11.35",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "spgispeech-wer",
    "score": 0.0451,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (short-form SPGI column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "SPGI",
      "raw_value": "4.51",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "earnings21-10m-wer",
    "score": 0.1009,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (long-form E21 10m column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "E21 10m",
      "raw_value": "10.09",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "earnings22-10m-wer",
    "score": 0.1227,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Mistral AI's evaluation of baseline 'GPT4o mini Transcribe' in Table 3 (long-form E22 10m column). WER as a fraction (percent divided by 100). Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T3",
      "caption": "Table 3: English speech recognition results by task.",
      "row_label": "GPT4o mini Transcribe",
      "column_label": "E22 10m",
      "raw_value": "12.27",
      "scale_divisor": 100
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.04,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Table 4 (per-language FLEURS WER), row 'Whisper large-v3', column 'en'. WER as a fraction. Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T4",
      "caption": "Table 4: Per-language WER scores for FLEURS ...",
      "row_label": "Whisper large-v3",
      "column_label": "en",
      "raw_value": "4.00",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "fleurs-en-wer",
    "score": 0.0319,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "independent",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Table 4 (per-language FLEURS WER), row 'GPT-4o mini Transcribe', column 'en'. WER as a fraction. Provenance: Mistral's own runs — §4 'We compare the model to GPT-4o mini Audio (/Transcribe) and Gemini 2.5 Flash, as well as Scribe and Whisper large-v3 on speech recognition tasks'; §1 'During evaluation of Voxtral and other models...'; the E21/E22 10-minute segmentations are Mistral's own eval sets (HF collection mistralai/speech-evals) and are not published elsewhere. 'ElevenLabs Scribe', 'Gemini 2.5 Flash' and 'Voxtral Mini Transcribe' rows not used (unversioned / LLM / API model outside the catalogue).",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T4",
      "caption": "Table 4: Per-language WER scores for FLEURS ...",
      "row_label": "GPT-4o mini Transcribe",
      "column_label": "en",
      "raw_value": "3.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "fleurs-en-wer",
    "score": 0.0377,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Table 4 (per-language FLEURS WER), row 'Voxtral Mini', column 'en'. WER as a fraction. Mistral AI's own evaluation of its open-weights 2507 checkpoint.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T4",
      "caption": "Table 4: Per-language WER scores for FLEURS ...",
      "row_label": "Voxtral Mini",
      "column_label": "en",
      "raw_value": "3.77",
      "scale_divisor": 100
    }
  },
  {
    "model": "voxtral-small-24b",
    "bench": "fleurs-en-wer",
    "score": 0.0335,
    "source": "https://arxiv.org/html/2507.13264v1",
    "source_id": "report-2507.13264v1",
    "by": "lab",
    "date": "2025-07-17",
    "mode": "transcription",
    "note": "Table 4 (per-language FLEURS WER), row 'Voxtral Small', column 'en'. WER as a fraction. Mistral AI's own evaluation of its open-weights 2507 checkpoint.",
    "evidence": {
      "kind": "table",
      "table_id": "A1.T4",
      "caption": "Table 4: Per-language WER scores for FLEURS ...",
      "row_label": "Voxtral Small",
      "column_label": "en",
      "raw_value": "3.35",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0139,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 1. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "1.39 | 3.75",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "librispeech-test-other-wer",
    "score": 0.0375,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 2. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "1.39 | 3.75",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0151,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 1. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Whisper -large-v3",
      "raw_value": "1.51 | 3.97",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0397,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 2. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Whisper -large-v3",
      "raw_value": "1.51 | 3.97",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0211,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 1. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "2.11 | 4.55",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0455,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'Librispeech clean | other' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'clean | other', part 2. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Librispeech clean | other",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "2.11 | 4.55",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "gigaspeech-wer",
    "score": 0.255,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'GigaSpeech' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "GigaSpeech",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "25.50",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeech-wer",
    "score": 0.0976,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'GigaSpeech' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "GigaSpeech",
      "column_label": "Whisper -large-v3",
      "raw_value": "9.76",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "gigaspeech-wer",
    "score": 0.0888,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'GigaSpeech' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "GigaSpeech",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "8.88",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.0908,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'CV-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-en",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "9.08",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.099,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'CV-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-en",
      "column_label": "Whisper -large-v3",
      "raw_value": "9.90",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.0992,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'CV-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-en",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "9.92",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "fleurs-en-wer",
    "score": 0.024,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'Fleurs-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-en",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "2.40",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0408,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'Fleurs-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-en",
      "column_label": "Whisper -large-v3",
      "raw_value": "4.08",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0439,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'Fleurs-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-en",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "4.39",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "mls-en-wer",
    "score": 0.0512,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'MLS-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "MLS-en",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "5.12",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "mls-en-wer",
    "score": 0.0487,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'MLS-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "MLS-en",
      "column_label": "Whisper -large-v3",
      "raw_value": "4.87",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "mls-en-wer",
    "score": 0.06,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'MLS-en' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "MLS-en",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "6.00",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "tedlium-wer",
    "score": 0.0769,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'Tedlium' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Tedlium",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "7.69",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "tedlium-wer",
    "score": 0.0684,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'Tedlium' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Tedlium",
      "column_label": "Whisper -large-v3",
      "raw_value": "6.84",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "tedlium-wer",
    "score": 0.0385,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'Tedlium' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Tedlium",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "3.85",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "voxpopuli-en-wer",
    "score": 0.1029,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'VoxPopuli' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "VoxPopuli",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "10.29",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "voxpopuli-en-wer",
    "score": 0.1205,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'VoxPopuli' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "VoxPopuli",
      "column_label": "Whisper -large-v3",
      "raw_value": "12.05",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "voxpopuli-en-wer",
    "score": 0.0996,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'VoxPopuli' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "VoxPopuli",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "9.96",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "wenetspeech-test-net-err",
    "score": 0.153,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 1. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "15.30 | 32.27",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.3227,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 2. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "15.30 | 32.27",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "wenetspeech-test-net-err",
    "score": 0.0986,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 1. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Whisper -large-v3",
      "raw_value": "9.86 | 19.11",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.1911,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 2. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Whisper -large-v3",
      "raw_value": "9.86 | 19.11",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "wenetspeech-test-net-err",
    "score": 0.0597,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 1. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "5.97 | 6.88",
      "scale_divisor": 100,
      "cell_part": 0
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.0688,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'WenetSpeech net | meeting' of 'ASR Benchmarks on Public Datasets (WER ↓)'; cell reports 'net | meeting', part 2. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "WenetSpeech net | meeting",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "5.97 | 6.88",
      "scale_divisor": 100,
      "cell_part": 1
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "aishell2-test-err",
    "score": 0.0424,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'AISHELL-2-test' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "AISHELL-2-test",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "4.24",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "aishell2-test-err",
    "score": 0.0506,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'AISHELL-2-test' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "AISHELL-2-test",
      "column_label": "Whisper -large-v3",
      "raw_value": "5.06",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "aishell2-test-err",
    "score": 0.0315,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'AISHELL-2-test' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "AISHELL-2-test",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "3.15",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "fleurs-zh-err",
    "score": 0.0244,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'Fleurs-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-zh",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "2.44",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-zh-err",
    "score": 0.0409,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'Fleurs-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-zh",
      "column_label": "Whisper -large-v3",
      "raw_value": "4.09",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "fleurs-zh-err",
    "score": 0.0288,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'Fleurs-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "Fleurs-zh",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "2.88",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "commonvoice-zh-err-version-unspecified",
    "score": 0.0632,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'GPT-4o -Transcribe', row 'CV-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-zh",
      "column_label": "GPT-4o -Transcribe",
      "raw_value": "6.32",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "commonvoice-zh-err-version-unspecified",
    "score": 0.1291,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "independent",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Whisper -large-v3', row 'CV-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Baseline run by the Qwen team. Qwen3-ASR Technical Report (arXiv:2601.21337v2, CC BY 4.0), Table 3 caption, identical table: 'For the commercial APIs and the open-source Whisper-large-v3 model, we obtained results by running inference on the test sets ourselves due to the absence of published numbers; for FunASR-MLT-Nano, we report the results from its official technical report.' Gemini-2.5-Pro (an LLM), Doubao-ASR (unversioned) and the copied FunASR-MLT-Nano column are not used.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-zh",
      "column_label": "Whisper -large-v3",
      "raw_value": "12.91",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "commonvoice-zh-err-version-unspecified",
    "score": 0.0689,
    "source": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md",
    "source_id": "card2-qwen-qwen3-asr-1-7b",
    "by": "lab",
    "date": "2026-01-30",
    "note": "Qwen team's evaluation, column 'Qwen3-ASR -0.6B', row 'CV-zh' of 'ASR Benchmarks on Public Datasets (WER ↓)'. Converted from percent. Qwen's own open-weights 0.6B checkpoint: vLLM, bfloat16, max_new_tokens=1024, greedy, no language parameter.",
    "evidence": {
      "kind": "table",
      "section": "Evaluation",
      "row_label": "CV-zh",
      "column_label": "Qwen3-ASR -0.6B",
      "raw_value": "6.89",
      "scale_divisor": 100,
      "cell_part": null
    }
  },
  {
    "model": "cogvideox-5b",
    "bench": "genai-arena-t2v",
    "score": 1125.6,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1125.649374550947 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: fal endpoint 'cogvideox-5b' (model/models/fal_api_models.py) = THUDM/CogVideoX-5b.",
    "evidence": {
      "source_model_label": "CogVideoX-5B",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1125.649374550947",
      "divisor": 1,
      "source_row": [
        "CogVideoX-5B",
        "CogVideoX-5B",
        "1125.649374550947",
        "1127.9999945471436",
        "CogVideoX LICENSE",
        "THUDM",
        "https://github.com/THUDM/CogVideo"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "fal endpoint 'cogvideox-5b' (model/models/fal_api_models.py) = THUDM/CogVideoX-5b",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "mochi-1",
    "bench": "genai-arena-t2v",
    "score": 1121.3,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1121.3241004152032 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub arena branch infermodels/mochi.py loads genmo/mochi-1-preview.",
    "evidence": {
      "source_model_label": "Mochi1",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1121.3241004152032",
      "divisor": 1,
      "source_row": [
        "Mochi1",
        "Mochi1",
        "1121.3241004152032",
        "1121.4458082183237",
        "Apache 2.0",
        "Genmo AI",
        "https://github.com/genmoai/mochi"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub arena branch infermodels/mochi.py loads genmo/mochi-1-preview",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "pyramid-flow-sd3",
    "bench": "genai-arena-t2v",
    "score": 1113.9,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1113.908975225855 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub arena infermodels/pyramid_flow.py: rain1011/pyramid-flow-sd3, model_variant diffusion_transformer_768p, 1280x768 (file unchanged since 2024-10-19).",
    "evidence": {
      "source_model_label": "Pyramid Flow",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1113.908975225855",
      "divisor": 1,
      "source_row": [
        "Pyramid Flow",
        "Pyramid Flow",
        "1113.908975225855",
        "1115.7084017633094",
        "MIT LICENSE",
        "Peking University",
        "https://pyramid-flow.github.io/"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub arena infermodels/pyramid_flow.py: rain1011/pyramid-flow-sd3, model_variant diffusion_transformer_768p, 1280x768 (file unchanged since 2024-10-19)",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "t2v-turbo-vc2",
    "bench": "genai-arena-t2v",
    "score": 1054.7,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "lab",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1054.667346620255 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: Space registry fal_T2VTurbo_text2video -> fal endpoint 't2v-turbo'; earlier VideoGenHub path loads jiachenli-ucsb/T2V-Turbo-VC2. Lab: Wenhu Chen co-authored both T2V-Turbo and GenAI-Arena.",
    "evidence": {
      "source_model_label": "T2V-Turbo",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1054.667346620255",
      "divisor": 1,
      "source_row": [
        "T2V-Turbo",
        "T2V-Turbo",
        "1054.667346620255",
        "1055.1178159763037",
        "cc-by-nc-4.0",
        "University of California, Santa Barbara",
        "https://github.com/Ji4chenLi/t2v-turbo"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "Space registry fal_T2VTurbo_text2video -> fal endpoint 't2v-turbo'; earlier VideoGenHub path loads jiachenli-ucsb/T2V-Turbo-VC2",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "cogvideox-2b",
    "bench": "genai-arena-t2v",
    "score": 1053.5,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1053.5275294595008 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub arena infermodels/cogvideox.py default weight THUDM/CogVideoX-2b.",
    "evidence": {
      "source_model_label": "CogVideoX-2B",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1053.5275294595008",
      "divisor": 1,
      "source_row": [
        "CogVideoX-2B",
        "CogVideoX-2B",
        "1053.5275294595008",
        "1051.844749266328",
        "CogVideoX LICENSE",
        "THUDM",
        "https://github.com/THUDM/CogVideo"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub arena infermodels/cogvideox.py default weight THUDM/CogVideoX-2b",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "genai-arena-t2v",
    "score": 1029.2,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1029.2459035505656 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub infermodels/videocrafter.py: VideoCrafter/VideoCrafter2 model.ckpt.",
    "evidence": {
      "source_model_label": "VideoCrafter2",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1029.2459035505656",
      "divisor": 1,
      "source_row": [
        "VideoCrafter2",
        "VideoCrafter2",
        "1029.2459035505656",
        "1028.6590294577172",
        "Apache 2.0",
        "Tencent AI Lab",
        "https://ailab-cvc.github.io/videocrafter2/"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub infermodels/videocrafter.py: VideoCrafter/VideoCrafter2 model.ckpt",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "allegro",
    "bench": "genai-arena-t2v",
    "score": 1001.9,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 1001.9093019926374 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub arena infermodels/allegro.py: rhymes-ai/Allegro (20 inference steps instead of the default 100).",
    "evidence": {
      "source_model_label": "Allegro",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "1001.9093019926374",
      "divisor": 1,
      "source_row": [
        "Allegro",
        "Allegro",
        "1001.9093019926374",
        "1004.4218404258124",
        "Apache 2.0",
        "rhymes-ai",
        "https://github.com/rhymes-ai/Allegro"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub arena infermodels/allegro.py: rhymes-ai/Allegro (20 inference steps instead of the default 100)",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "lavie",
    "bench": "genai-arena-t2v",
    "score": 964,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 964.0322321092232 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub infermodels/lavie.py: Vchitect/LaVie lavie_base.pt.",
    "evidence": {
      "source_model_label": "LaVie",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "964.0322321092232",
      "divisor": 1,
      "source_row": [
        "LaVie",
        "LaVie",
        "964.0322321092232",
        "963.5205166809438",
        "Apache 2.0",
        "Shanghai AI Lab",
        "https://github.com/Vchitect/LaVie"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub infermodels/lavie.py: Vchitect/LaVie lavie_base.pt",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "ltx-video-0-9",
    "bench": "genai-arena-t2v",
    "score": 964,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 963.9592713286161 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub arena infermodels/ltxvideo.py (added 2024-11-23): Lightricks/LTX-Video unet/vae/scheduler layout of the original 0.9 release.",
    "evidence": {
      "source_model_label": "LTXVideo",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "963.9592713286161",
      "divisor": 1,
      "source_row": [
        "LTXVideo",
        "LTXVideo",
        "963.9592713286161",
        "964.0987281313567",
        "Apache 2.0",
        "Lightricks",
        "https://github.com/Lightricks/LTX-Video"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub arena infermodels/ltxvideo.py (added 2024-11-23): Lightricks/LTX-Video unet/vae/scheduler layout of the original 0.9 release",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "open-sora-1-2",
    "bench": "genai-arena-t2v",
    "score": 871.5,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 871.4979371531214 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub infermodels/opensora_12.py: hpcai-tech/OpenSora-STDiT-v3 + OpenSora-VAE-v1.2.",
    "evidence": {
      "source_model_label": "OpenSora v1.2",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "871.4979371531214",
      "divisor": 1,
      "source_row": [
        "OpenSora v1.2",
        "OpenSora v1.2",
        "871.4979371531214",
        "869.286544067318",
        "Apache 2.0",
        "HPC-AI Tech",
        "https://github.com/hpcaitech/Open-Sora"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub infermodels/opensora_12.py: hpcai-tech/OpenSora-STDiT-v3 + OpenSora-VAE-v1.2",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "genai-arena-t2v",
    "score": 830.4,
    "source": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv",
    "source_id": "video-genai-arena-space",
    "source_group": "video-genai-arena-space:genai-arena-t2v",
    "evaluator": "GenAI-Arena (TIGER-Lab, University of Waterloo) public anonymous pairwise votes",
    "by": "independent",
    "date": "2025-03-22",
    "note": "GenAI-Arena text-to-video leaderboard snapshot arena_elo/results/20250322 at Space revision 4197b3a; 'Arena Elo rating (anony)' 830.3604437990733 (anonymous votes only, the column the Space displays); rounded to 0.1. Identity: VideoGenHub infermodels/modelscope.py: ali-vilab/modelscope-damo-text-to-video-synthesis.",
    "evidence": {
      "source_model_label": "ModelScope",
      "table": "arena_elo/results/20250322/video_generation_leaderboard.csv",
      "reported_value": "830.3604437990733",
      "divisor": 1,
      "source_row": [
        "ModelScope",
        "ModelScope",
        "830.3604437990733",
        "831.0034736422567",
        "cc-by-nc-4.0",
        "Alibaba Group",
        "https://arxiv.org/abs/2308.06571"
      ],
      "source_header": [
        "key",
        "Model",
        "Arena Elo rating (anony)",
        "Arena Elo rating (full)",
        "License",
        "Organization",
        "Link"
      ],
      "measurement_origin": "public anonymous pairwise human votes collected by the GenAI-Arena platform; Elo computed by the maintainers' arena_elo scripts",
      "identity_evidence": "VideoGenHub infermodels/modelscope.py: ali-vilab/modelscope-damo-text-to-video-synthesis",
      "date_basis": "results folder date 2025-03-22 (identical bytes in results/latest at this revision)",
      "source_artifact_sha256": "32a769b5f80093ebe7b2e145f1efa912593b85f847f966a6e63e161febac409e"
    }
  },
  {
    "model": "cogvideox-5b",
    "bench": "videophy-human-joint",
    "score": 0.396,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 39.6% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: row links THUDM/CogVideo; paper lists CogVideoX-5B.",
    "evidence": {
      "source_model_label": "CogVideoX-5B",
      "row_link": "https://github.com/THUDM/CogVideo",
      "table": "README.md Human LeaderBoard",
      "reported_value": "39.6",
      "divisor": 100,
      "source_row": [
        "1",
        "[CogVideoX-5B](https://github.com/THUDM/CogVideo)🥇",
        "Open",
        "**39.6**",
        "**63.3**",
        "**53**"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links THUDM/CogVideo; paper lists CogVideoX-5B",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "pika-1-0",
    "bench": "videophy-human-joint",
    "score": 0.197,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 19.7% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: VideoPhy v1 (arXiv 2406.03520v1, 5 Jun 2024) already reports Pika at 19.7% (same cell), i.e. the Pika 1.0 era product (Pika 1.5 launched Oct 2024); Appendix K: 640x1088, 72 frames.",
    "evidence": {
      "source_model_label": "Pika",
      "row_link": "https://pika.art/",
      "table": "README.md Human LeaderBoard",
      "reported_value": "19.7",
      "divisor": 100,
      "source_row": [
        "2",
        "[Pika](https://pika.art/)🥈",
        "Closed",
        "19.7",
        "41.1",
        "36.5"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "VideoPhy v1 (arXiv 2406.03520v1, 5 Jun 2024) already reports Pika at 19.7% (same cell), i.e. the Pika 1.0 era product (Pika 1.5 launched Oct 2024); Appendix K: 640x1088, 72 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "videophy-human-joint",
    "score": 0.19,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 19.0% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: row links VideoCrafter2 paper; Appendix K: 320x512.",
    "evidence": {
      "source_model_label": "VideoCrafter2",
      "row_link": "https://arxiv.org/abs/2401.09047",
      "table": "README.md Human LeaderBoard",
      "reported_value": "19.0",
      "divisor": 100,
      "source_row": [
        "3",
        "[VideoCrafter2](https://arxiv.org/abs/2401.09047)🥉",
        "Open",
        "19.0",
        "48.5",
        "34.6"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links VideoCrafter2 paper; Appendix K: 320x512",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "cogvideox-2b",
    "bench": "videophy-human-joint",
    "score": 0.186,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 18.6% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: row links THUDM/CogVideo; paper lists CogVideoX-2B.",
    "evidence": {
      "source_model_label": "CogVideoX-2B",
      "row_link": "https://github.com/THUDM/CogVideo",
      "table": "README.md Human LeaderBoard",
      "reported_value": "18.6",
      "divisor": 100,
      "source_row": [
        "4",
        "[CogVideoX-2B](https://github.com/THUDM/CogVideo)",
        "Open",
        "18.6",
        "47.2",
        "34.1"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links THUDM/CogVideo; paper lists CogVideoX-2B",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "lavie",
    "bench": "videophy-human-joint",
    "score": 0.157,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 15.7% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: row links LaVie paper; Appendix K: 320x512 (LaVie base).",
    "evidence": {
      "source_model_label": "LaVIE",
      "row_link": "https://arxiv.org/abs/2309.15103",
      "table": "README.md Human LeaderBoard",
      "reported_value": "15.7",
      "divisor": 100,
      "source_row": [
        "5",
        "[LaVIE](https://arxiv.org/abs/2309.15103)",
        "Open",
        "15.7",
        "48.7",
        "28.0"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links LaVie paper; Appendix K: 320x512 (LaVie base)",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "zeroscope-v2-576w",
    "bench": "videophy-human-joint",
    "score": 0.119,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 11.9% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: row links huggingface.co/cerspense/zeroscope_v2_576w; Appendix K: 320x576.",
    "evidence": {
      "source_model_label": "ZeroScope",
      "row_link": "https://huggingface.co/cerspense/zeroscope_v2_576w",
      "table": "README.md Human LeaderBoard",
      "reported_value": "11.9",
      "divisor": 100,
      "source_row": [
        "9",
        "[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w)",
        "Open",
        "11.9",
        "30.2",
        "32.6"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links huggingface.co/cerspense/zeroscope_v2_576w; Appendix K: 320x576",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "lumiere-t2v",
    "bench": "videophy-human-joint",
    "score": 0.09,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 9.0% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: Google Lumiere text-to-video (closed; generated by the Lumiere authors per paper); Appendix K: 1024x1024, 80 frames.",
    "evidence": {
      "source_model_label": "Lumiere-T2V",
      "row_link": "https://arxiv.org/abs/2401.12945",
      "table": "README.md Human LeaderBoard",
      "reported_value": "9.0",
      "divisor": 100,
      "source_row": [
        "10",
        "[Lumiere-T2V](https://arxiv.org/abs/2401.12945)",
        "Closed",
        "9.0",
        "38.4",
        "27.9"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "Google Lumiere text-to-video (closed; generated by the Lumiere authors per paper); Appendix K: 1024x1024, 80 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "runway-gen-2",
    "bench": "videophy-human-joint",
    "score": 0.076,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 human leaderboard, 'SA=1, PC=1' 7.6% (joint semantic adherence and physical commonsense, human annotation). README.md at commit 01d366c. Identity: Gen-2 (Runway) evaluated in VideoPhy v1 (Jun 2024); Appendix K: 720x1280, 32 frames.",
    "evidence": {
      "source_model_label": "Gen-2 (Runway)",
      "row_link": "https://arxiv.org/abs/2302.03011",
      "table": "README.md Human LeaderBoard",
      "reported_value": "7.6",
      "divisor": 100,
      "source_row": [
        "11",
        "[Gen-2 (Runway)](https://arxiv.org/abs/2302.03011)",
        "Closed",
        "7.6",
        "26.6",
        "27.2"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**SA=1, PC=1**",
        "**SA=1**",
        "**PC=1**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "Gen-2 (Runway) evaluated in VideoPhy v1 (Jun 2024); Appendix K: 720x1280, 32 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "videophy-autoeval-avg",
    "score": 0.41,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 41%. README.md at commit 01d366c. Identity: row links VideoCrafter2 paper; Appendix K: 320x512.",
    "evidence": {
      "source_model_label": "VideoCrafter2",
      "row_link": "https://arxiv.org/abs/2401.09047",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "41",
      "divisor": 100,
      "source_row": [
        "3",
        "[VideoCrafter2](https://arxiv.org/abs/2401.09047)🥉",
        "36",
        "47",
        "41"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links VideoCrafter2 paper; Appendix K: 320x512",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "lavie",
    "bench": "videophy-autoeval-avg",
    "score": 0.41,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 41%. README.md at commit 01d366c. Identity: row links LaVie paper; Appendix K: 320x512 (LaVie base).",
    "evidence": {
      "source_model_label": "LaVIE",
      "row_link": "https://arxiv.org/abs/2309.15103",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "41",
      "divisor": 100,
      "source_row": [
        "4",
        "[LaVIE](https://arxiv.org/abs/2309.15103)",
        "36",
        "45",
        "41"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links LaVie paper; Appendix K: 320x512 (LaVie base)",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "cogvideox-2b",
    "bench": "videophy-autoeval-avg",
    "score": 0.39,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 39%. README.md at commit 01d366c. Identity: row links THUDM/CogVideo; paper lists CogVideoX-2B.",
    "evidence": {
      "source_model_label": "CogVideoX-2B",
      "row_link": "https://github.com/THUDM/CogVideo",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "39",
      "divisor": 100,
      "source_row": [
        "6",
        "[CogVideoX-2B](https://github.com/THUDM/CogVideo)",
        "39",
        "40",
        "39"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links THUDM/CogVideo; paper lists CogVideoX-2B",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "zeroscope-v2-576w",
    "bench": "videophy-autoeval-avg",
    "score": 0.34,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 34%. README.md at commit 01d366c. Identity: row links huggingface.co/cerspense/zeroscope_v2_576w; Appendix K: 320x576.",
    "evidence": {
      "source_model_label": "ZeroScope",
      "row_link": "https://huggingface.co/cerspense/zeroscope_v2_576w",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "34",
      "divisor": 100,
      "source_row": [
        "10",
        "[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w)",
        "42",
        "27",
        "34"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "row links huggingface.co/cerspense/zeroscope_v2_576w; Appendix K: 320x576",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "lumiere-t2v",
    "bench": "videophy-autoeval-avg",
    "score": 0.33,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 33%. README.md at commit 01d366c. Identity: Google Lumiere text-to-video (closed; generated by the Lumiere authors per paper); Appendix K: 1024x1024, 80 frames.",
    "evidence": {
      "source_model_label": "Lumiere-T2V",
      "row_link": "https://arxiv.org/abs/2401.12945",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "33",
      "divisor": 100,
      "source_row": [
        "11",
        "[Lumiere-T2V](https://arxiv.org/abs/2401.12945)",
        "31",
        "35",
        "33"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "Google Lumiere text-to-video (closed; generated by the Lumiere authors per paper); Appendix K: 1024x1024, 80 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "pika-1-0",
    "bench": "videophy-autoeval-avg",
    "score": 0.29,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 29%. README.md at commit 01d366c. Identity: VideoPhy v1 (arXiv 2406.03520v1, 5 Jun 2024) already reports Pika at 19.7% (same cell), i.e. the Pika 1.0 era product (Pika 1.5 launched Oct 2024); Appendix K: 640x1088, 72 frames.",
    "evidence": {
      "source_model_label": "Pika",
      "row_link": "https://pika.art/",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "29",
      "divisor": 100,
      "source_row": [
        "12",
        "[Pika](https://pika.art/)",
        "33",
        "25",
        "29"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "VideoPhy v1 (arXiv 2406.03520v1, 5 Jun 2024) already reports Pika at 19.7% (same cell), i.e. the Pika 1.0 era product (Pika 1.5 launched Oct 2024); Appendix K: 640x1088, 72 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "runway-gen-2",
    "bench": "videophy-autoeval-avg",
    "score": 0.29,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md#automatic-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy-autoeval-avg",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoPhy-1 automatic leaderboard (VideoCon-Physics auto-evaluator), Avg. of PC=1 and SA=1 = 29%. README.md at commit 01d366c. Identity: Gen-2 (Runway) evaluated in VideoPhy v1 (Jun 2024); Appendix K: 720x1280, 32 frames.",
    "evidence": {
      "source_model_label": "Gen-2 (Runway)",
      "row_link": "https://arxiv.org/abs/2302.03011",
      "table": "README.md Automatic LeaderBoard",
      "reported_value": "29",
      "divisor": 100,
      "source_row": [
        "13",
        "[Gen-2 (Runway)](https://arxiv.org/abs/2302.03011)",
        "31",
        "26",
        "29"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**PC=1**",
        "**SA=1**",
        "**Avg.**"
      ],
      "measurement_origin": "benchmark authors' own auto-evaluator run on the generated test videos",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "identity_evidence": "Gen-2 (Runway) evaluated in VideoPhy v1 (Jun 2024); Appendix K: 720x1280, 32 frames",
      "source_artifact_sha256": "b849ff1b8936cc29b4e31a5ca1998622c6116cf9ea7ee10f5dee067e1e345e57"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "videophy2-human-joint",
    "score": 0.105,
    "source": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md#human-leaderboard-",
    "source_id": "visual-videophy-readme",
    "source_group": "visual-videophy-readme:videophy2-human-joint",
    "evaluator": "VideoPhy benchmark authors (Hritik Bansal et al.)",
    "by": "independent",
    "date": "2026-01-30",
    "note": "VideoCrafter2; VideoPhy-2 human leaderboard 'All' 10.5% (SA>=4 and PC>=4); row links huggingface.co/VideoCrafter/VideoCrafter2. VIDEOPHY2/README.md at commit 01d366c.",
    "evidence": {
      "source_model_label": "VideoCrafter2",
      "row_link": "https://huggingface.co/VideoCrafter/VideoCrafter2",
      "table": "VIDEOPHY2/README.md leaderboard",
      "reported_value": "10.5",
      "divisor": 100,
      "source_row": [
        "7",
        "[VideoCrafter2](https://huggingface.co/VideoCrafter/VideoCrafter2)",
        "Open",
        "10.5",
        "2.9"
      ],
      "source_header": [
        "**#**",
        "**Model**",
        "**Source**",
        "**All**",
        "**Hard**"
      ],
      "measurement_origin": "explicit original human annotation by the benchmark authors",
      "date_basis": "Pinned repository commit date (2026-01-30), not an asserted generation date",
      "source_artifact_sha256": "42b9fcc72e7c68cb054df1f23ed56241213ca36146b4982111f05870baabd2e6"
    }
  },
  {
    "model": "sora",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.6209,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 62.09 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7: Sora web product (ref [69], accessed 2025-03-08), 854x480, 150 frames, 30 fps (representative settings).",
    "evidence": {
      "source_model_label": "Sora [ 69 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "62.09",
      "divisor": 100,
      "source_row": [
        "Sora [ 69 ]",
        "64.10",
        "61.79",
        "61.22",
        "63.04",
        "62.66",
        "61.97",
        "57.73",
        "62.77",
        "63.82",
        "64.31",
        "62.01",
        "63.13",
        "62.29",
        "62.20",
        "62.71",
        "63.90",
        "62.62",
        "59.00",
        "57.96",
        "62.89",
        "62.09",
        "3"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7: Sora web product (ref [69], accessed 2025-03-08), 854x480, 150 frames, 30 fps (representative settings)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "runway-gen-3-alpha",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.5922,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 59.22 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 'Gen3' cites ref [71] 'Introducing Gen-3 Alpha' (accessed 2025-03-08), 1280x768, 128 frames, 24 fps.",
    "evidence": {
      "source_model_label": "Gen3 [ 71 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "59.22",
      "divisor": 100,
      "source_row": [
        "Gen3 [ 71 ]",
        "58.22",
        "62.37",
        "60.26",
        "55.98",
        "57.42",
        "58.36",
        "61.61",
        "65.00",
        "56.84",
        "61.68",
        "59.16",
        "57.86",
        "53.78",
        "59.41",
        "58.34",
        "58.71",
        "59.21",
        "58.97",
        "56.12",
        "62.53",
        "59.22",
        "7"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 'Gen3' cites ref [71] 'Introducing Gen-3 Alpha' (accessed 2025-03-08), 1280x768, 128 frames, 24 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "allegro",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.5608,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 56.08 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/rhymes-ai/Allegro, 1280x720, 88 frames, 15 fps (Allegro T2V defaults).",
    "evidence": {
      "source_model_label": "Allegro [ 78 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "56.08",
      "divisor": 100,
      "source_row": [
        "Allegro [ 78 ]",
        "60.43",
        "54.95",
        "54.82",
        "55.32",
        "52.69",
        "60.28",
        "50.11",
        "62.52",
        "61.63",
        "40.30",
        "58.22",
        "51.73",
        "52.17",
        "61.43",
        "58.75",
        "50.78",
        "61.53",
        "52.54",
        "49.27",
        "55.36",
        "56.08",
        "14"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/rhymes-ai/Allegro, 1280x720, 88 frames, 15 fps (Allegro T2V defaults)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "vidu-1-5",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.5456,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 54.56 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 label 'Vidu1.5' (ref [39], vidu.studio, accessed 2025-03-08), 688x384, 60 frames, 16 fps.",
    "evidence": {
      "source_model_label": "Vidu1.5 [ 39 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "54.56",
      "divisor": 100,
      "source_row": [
        "Vidu1.5 [ 39 ]",
        "56.09",
        "55.93",
        "53.52",
        "57.93",
        "56.07",
        "51.97",
        "59.85",
        "53.51",
        "53.29",
        "55.28",
        "51.32",
        "52.42",
        "56.23",
        "55.83",
        "50.58",
        "55.71",
        "56.74",
        "50.59",
        "51.55",
        "58.78",
        "54.56",
        "15"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 label 'Vidu1.5' (ref [39], vidu.studio, accessed 2025-03-08), 688x384, 60 frames, 16 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.5059,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 50.59 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 'CogVideo X1.5' (github.com/THUDM/CogVideo), 1360x768 (CogVideoX1.5-5B native resolution), 32 frames, 8 fps.",
    "evidence": {
      "source_model_label": "CogVideo X1.5 [ 80 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "50.59",
      "divisor": 100,
      "source_row": [
        "CogVideo X1.5 [ 80 ]",
        "54.63",
        "56.76",
        "53.80",
        "48.81",
        "45.18",
        "53.31",
        "43.78",
        "53.96",
        "48.30",
        "38.60",
        "51.09",
        "42.75",
        "47.11",
        "54.83",
        "59.13",
        "48.34",
        "53.75",
        "47.53",
        "44.12",
        "49.81",
        "50.59",
        "16"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 'CogVideo X1.5' (github.com/THUDM/CogVideo), 1360x768 (CogVideoX1.5-5B native resolution), 32 frames, 8 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "lavie",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.493,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 49.30 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/Vchitect/LaVie, 512x320, 16 frames, 8 fps (LaVie base).",
    "evidence": {
      "source_model_label": "Lavie [ 82 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "49.30",
      "divisor": 100,
      "source_row": [
        "Lavie [ 82 ]",
        "54.33",
        "52.79",
        "49.66",
        "48.20",
        "42.82",
        "47.63",
        "46.07",
        "52.62",
        "51.21",
        "46.48",
        "48.44",
        "46.87",
        "46.83",
        "53.18",
        "48.05",
        "50.79",
        "50.75",
        "47.54",
        "45.19",
        "50.90",
        "49.30",
        "18"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/Vchitect/LaVie, 512x320, 16 frames, 8 fps (LaVie base)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.4811,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 48.11 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/AILab-CVC/VideoCrafter, cites VideoCrafter2 paper [79], 512x320, 16 frames.",
    "evidence": {
      "source_model_label": "VideoCrafter2 [ 79 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "48.11",
      "divisor": 100,
      "source_row": [
        "VideoCrafter2 [ 79 ]",
        "48.40",
        "50.89",
        "46.50",
        "51.81",
        "44.09",
        "47.06",
        "50.27",
        "46.31",
        "51.24",
        "45.92",
        "49.16",
        "52.89",
        "45.08",
        "48.61",
        "45.25",
        "51.45",
        "47.22",
        "48.99",
        "43.02",
        "47.05",
        "48.11",
        "19"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/AILab-CVC/VideoCrafter, cites VideoCrafter2 paper [79], 512x320, 16 frames",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "hotshot-xl",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.4266,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 42.66 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/hotshotco/Hotshot-XL, 672x384, 8 frames, 8 fps.",
    "evidence": {
      "source_model_label": "Hotshot-XL [ 83 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "42.66",
      "divisor": 100,
      "source_row": [
        "Hotshot-XL [ 83 ]",
        "45.19",
        "46.73",
        "40.55",
        "35.37",
        "33.92",
        "42.75",
        "40.27",
        "42.37",
        "41.59",
        "50.01",
        "42.78",
        "45.72",
        "41.01",
        "48.17",
        "37.79",
        "39.45",
        "43.61",
        "42.65",
        "36.38",
        "44.20",
        "42.66",
        "22"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/hotshotco/Hotshot-XL, 672x384, 8 frames, 8 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.38,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 38.00 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/modelscope/modelscope, cites ModelScope T2V report [88], 256x256, 16 frames, 8 fps.",
    "evidence": {
      "source_model_label": "ModelScope [ 88 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "38.00",
      "divisor": 100,
      "source_row": [
        "ModelScope [ 88 ]",
        "43.82",
        "42.94",
        "37.64",
        "36.89",
        "37.25",
        "38.97",
        "33.68",
        "39.16",
        "34.97",
        "36.79",
        "36.41",
        "39.56",
        "34.09",
        "39.73",
        "34.50",
        "34.53",
        "37.54",
        "35.96",
        "36.48",
        "40.78",
        "38.00",
        "26"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/modelscope/modelscope, cites ModelScope T2V report [88], 256x256, 16 frames, 8 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "sora",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.5968,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 59.68 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7: Sora web product (ref [69], accessed 2025-03-08), 854x480, 150 frames, 30 fps (representative settings).",
    "evidence": {
      "source_model_label": "Sora [ 69 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "59.68",
      "divisor": 100,
      "source_row": [
        "Sora [ 69 ]",
        "63.12",
        "59.98",
        "56.67",
        "62.97",
        "52.07",
        "60.29",
        "55.57",
        "62.15",
        "64.17",
        "63.83",
        "61.50",
        "60.26",
        "56.38",
        "62.72",
        "60.60",
        "65.81",
        "62.48",
        "52.76",
        "46.76",
        "57.84",
        "59.68",
        "4"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7: Sora web product (ref [69], accessed 2025-03-08), 854x480, 150 frames, 30 fps (representative settings)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "vidu-1-5",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.5825,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 58.25 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 label 'Vidu1.5' (ref [39], vidu.studio, accessed 2025-03-08), 688x384, 60 frames, 16 fps.",
    "evidence": {
      "source_model_label": "Vidu1.5 [ 39 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "58.25",
      "divisor": 100,
      "source_row": [
        "Vidu1.5 [ 39 ]",
        "62.67",
        "61.91",
        "53.22",
        "59.30",
        "54.34",
        "59.00",
        "59.88",
        "57.97",
        "59.15",
        "47.33",
        "58.42",
        "60.87",
        "56.86",
        "59.48",
        "60.04",
        "56.40",
        "60.30",
        "55.39",
        "51.53",
        "60.42",
        "58.25",
        "5"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 label 'Vidu1.5' (ref [39], vidu.studio, accessed 2025-03-08), 688x384, 60 frames, 16 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "runway-gen-3-alpha",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.5572,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 55.72 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 'Gen3' cites ref [71] 'Introducing Gen-3 Alpha' (accessed 2025-03-08), 1280x768, 128 frames, 24 fps.",
    "evidence": {
      "source_model_label": "Gen3 [ 71 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "55.72",
      "divisor": 100,
      "source_row": [
        "Gen3 [ 71 ]",
        "54.94",
        "57.85",
        "45.91",
        "57.07",
        "49.53",
        "56.11",
        "58.16",
        "61.09",
        "53.52",
        "58.67",
        "60.25",
        "60.46",
        "48.55",
        "56.44",
        "56.39",
        "58.33",
        "59.43",
        "51.11",
        "38.83",
        "58.47",
        "55.72",
        "8"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 'Gen3' cites ref [71] 'Introducing Gen-3 Alpha' (accessed 2025-03-08), 1280x768, 128 frames, 24 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.5107,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 51.07 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/AILab-CVC/VideoCrafter, cites VideoCrafter2 paper [79], 512x320, 16 frames.",
    "evidence": {
      "source_model_label": "VideoCrafter2 [ 79 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "51.07",
      "divisor": 100,
      "source_row": [
        "VideoCrafter2 [ 79 ]",
        "53.29",
        "51.28",
        "45.34",
        "54.77",
        "41.20",
        "52.88",
        "52.08",
        "54.18",
        "57.15",
        "40.26",
        "54.19",
        "58.10",
        "51.08",
        "56.39",
        "53.94",
        "50.25",
        "50.05",
        "45.20",
        "39.26",
        "52.45",
        "51.07",
        "14"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/AILab-CVC/VideoCrafter, cites VideoCrafter2 paper [79], 512x320, 16 frames",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "allegro",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.5068,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 50.68 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/rhymes-ai/Allegro, 1280x720, 88 frames, 15 fps (Allegro T2V defaults).",
    "evidence": {
      "source_model_label": "Allegro [ 78 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "50.68",
      "divisor": 100,
      "source_row": [
        "Allegro [ 78 ]",
        "52.79",
        "50.36",
        "46.87",
        "49.98",
        "39.82",
        "53.12",
        "53.04",
        "54.19",
        "51.08",
        "32.03",
        "53.68",
        "46.06",
        "45.62",
        "58.72",
        "58.41",
        "40.39",
        "57.05",
        "50.19",
        "41.66",
        "54.12",
        "50.68",
        "15"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/rhymes-ai/Allegro, 1280x720, 88 frames, 15 fps (Allegro T2V defaults)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.4973,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 49.73 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 9 short label for the same CogVideo X1.5 row (Tables 3, 7, 8, 10 name it CogVideo X1.5).",
    "evidence": {
      "source_model_label": "CogVideo [ 80 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "49.73",
      "divisor": 100,
      "source_row": [
        "CogVideo [ 80 ]",
        "53.85",
        "50.62",
        "52.66",
        "48.11",
        "41.36",
        "55.25",
        "48.95",
        "55.62",
        "49.81",
        "35.12",
        "50.49",
        "47.68",
        "44.99",
        "52.92",
        "60.80",
        "41.26",
        "54.39",
        "43.37",
        "40.31",
        "49.84",
        "49.73",
        "17"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 9 short label for the same CogVideo X1.5 row (Tables 3, 7, 8, 10 name it CogVideo X1.5)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "lavie",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.4822,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 48.22 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/Vchitect/LaVie, 512x320, 16 frames, 8 fps (LaVie base).",
    "evidence": {
      "source_model_label": "Lavie [ 82 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "48.22",
      "divisor": 100,
      "source_row": [
        "Lavie [ 82 ]",
        "51.67",
        "45.55",
        "56.42",
        "51.23",
        "33.44",
        "49.08",
        "50.01",
        "51.64",
        "49.86",
        "39.97",
        "47.92",
        "48.77",
        "51.93",
        "52.13",
        "52.23",
        "47.33",
        "50.57",
        "42.31",
        "38.62",
        "50.07",
        "48.22",
        "19"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/Vchitect/LaVie, 512x320, 16 frames, 8 fps (LaVie base)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "hotshot-xl",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.4775,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 47.75 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/hotshotco/Hotshot-XL, 672x384, 8 frames, 8 fps.",
    "evidence": {
      "source_model_label": "Hotshot-XL [ 83 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "47.75",
      "divisor": 100,
      "source_row": [
        "Hotshot-XL [ 83 ]",
        "52.72",
        "49.17",
        "46.90",
        "41.35",
        "32.02",
        "49.99",
        "44.89",
        "48.26",
        "49.14",
        "52.34",
        "49.08",
        "54.94",
        "47.23",
        "52.36",
        "49.52",
        "38.66",
        "50.16",
        "41.62",
        "37.68",
        "50.67",
        "47.75",
        "20"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/hotshotco/Hotshot-XL, 672x384, 8 frames, 8 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.4373,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 43.73 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/modelscope/modelscope, cites ModelScope T2V report [88], 256x256, 16 frames, 8 fps.",
    "evidence": {
      "source_model_label": "ModelScope [ 88 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "43.73",
      "divisor": 100,
      "source_row": [
        "ModelScope [ 88 ]",
        "46.00",
        "46.29",
        "40.27",
        "45.07",
        "40.26",
        "46.29",
        "37.63",
        "46.23",
        "45.97",
        "41.93",
        "41.65",
        "53.65",
        "43.38",
        "41.21",
        "46.84",
        "41.24",
        "44.67",
        "40.09",
        "37.06",
        "46.05",
        "43.73",
        "25"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/modelscope/modelscope, cites ModelScope T2V report [88], 256x256, 16 frames, 8 fps",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "luma-dream-machine-1-6",
    "bench": "worldmodelbench-human-total",
    "score": 0.772,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 7.72 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning unstated (closed API). Table 3 of arXiv 2502.20694v1. Identity/settings: luma-1.6 (Section 4 model list), i.e. Luma Dream Machine 1.6 (Sep 2024).",
    "evidence": {
      "source_model_label": "Luma [ 35 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "7.72",
      "divisor": 10,
      "source_row": [
        "Luma [ 35 ]",
        "2.01",
        "0.81",
        "0.76",
        "0.89",
        "0.62",
        "0.95",
        "0.77",
        "0.90",
        "7.72"
      ],
      "conditioning": "unstated (closed API)",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "luma-1.6 (Section 4 model list), i.e. Luma Dream Machine 1.6 (Sep 2024)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "mochi-1",
    "bench": "worldmodelbench-human-total",
    "score": 0.762,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 7.62 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning text-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: open 'mochi' run locally, cfg 4.5, 65 steps (Appendix 8.3).",
    "evidence": {
      "source_model_label": "Mochi [ 3 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "7.62",
      "divisor": 10,
      "source_row": [
        "Mochi [ 3 ]",
        "2.22",
        "0.63",
        "0.63",
        "0.94",
        "0.58",
        "0.97",
        "0.71",
        "0.94",
        "7.62"
      ],
      "conditioning": "text-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "open 'mochi' run locally, cfg 4.5, 65 steps (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "open-sora-plan-1-3",
    "bench": "worldmodelbench-human-total",
    "score": 0.761,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 7.61 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning text-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: OpenSora-Plan-v1.3 T2V (Section 4); fps 18, cfg 7.5, 100 steps, 352x640 (Appendix 8.3).",
    "evidence": {
      "source_model_label": "OpenSoraPlan-T2V [ 28 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "7.61",
      "divisor": 10,
      "source_row": [
        "OpenSoraPlan-T2V [ 28 ]",
        "1.79",
        "0.70",
        "0.77",
        "0.9",
        "0.66",
        "0.97",
        "0.89",
        "0.93",
        "7.61"
      ],
      "conditioning": "text-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "OpenSora-Plan-v1.3 T2V (Section 4); fps 18, cfg 7.5, 100 steps, 352x640 (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "cogvideox-5b",
    "bench": "worldmodelbench-human-total",
    "score": 0.731,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 7.31 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning text-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: CogVideoX-5B T2V (Section 4); cfg 6.0, 50-step DDIM (Appendix 8.3).",
    "evidence": {
      "source_model_label": "CogVideoX-T2V [ 56 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "7.31",
      "divisor": 10,
      "source_row": [
        "CogVideoX-T2V [ 56 ]",
        "2.11",
        "0.60",
        "0.51",
        "0.91",
        "0.52",
        "0.96",
        "0.74",
        "0.95",
        "7.31"
      ],
      "conditioning": "text-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "CogVideoX-5B T2V (Section 4); cfg 6.0, 50-step DDIM (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "cogvideox-5b-i2v",
    "bench": "worldmodelbench-human-total",
    "score": 0.675,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 6.75 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning image-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: CogVideoX-5B I2V (Section 4); cfg 6.0, 50-step DDIM (Appendix 8.3).",
    "evidence": {
      "source_model_label": "CogVideoX-I2V [ 56 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "6.75",
      "divisor": 10,
      "source_row": [
        "CogVideoX-I2V [ 56 ]",
        "1.89",
        "0.56",
        "0.43",
        "0.87",
        "0.43",
        "0.96",
        "0.66",
        "0.96",
        "6.75"
      ],
      "conditioning": "image-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "CogVideoX-5B I2V (Section 4); cfg 6.0, 50-step DDIM (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "open-sora-plan-1-3-i2v",
    "bench": "worldmodelbench-human-total",
    "score": 0.662,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 6.62 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning image-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: OpenSora-Plan-v1.3 I2V (Section 4); fps 18, cfg 7.5, 100 steps, 352x640.",
    "evidence": {
      "source_model_label": "OpenSora-Plan-I2V [ 28 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "6.62",
      "divisor": 10,
      "source_row": [
        "OpenSora-Plan-I2V [ 28 ]",
        "1.77",
        "0.47",
        "0.54",
        "0.84",
        "0.42",
        "0.97",
        "0.70",
        "0.92",
        "6.62"
      ],
      "conditioning": "image-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "OpenSora-Plan-v1.3 I2V (Section 4); fps 18, cfg 7.5, 100 steps, 352x640",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "pandora",
    "bench": "worldmodelbench-human-total",
    "score": 0.657,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 6.57 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning image-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: Pandora official checkpoint, default settings, 50 DDIM steps (Appendix 8.3).",
    "evidence": {
      "source_model_label": "Pandora [ 53 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "6.57",
      "divisor": 10,
      "source_row": [
        "Pandora [ 53 ]",
        "1.56",
        "0.42",
        "0.53",
        "0.91",
        "0.50",
        "0.96",
        "0.74",
        "0.94",
        "6.57"
      ],
      "conditioning": "image-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "Pandora official checkpoint, default settings, 50 DDIM steps (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "t2v-turbo-v2",
    "bench": "worldmodelbench-human-total",
    "score": 0.622,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 6.22 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning text-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: T2VTurbo-v2 (Section 4); 4 steps, cfg 7.5, 16 frames at 16 fps (Appendix 8.3).",
    "evidence": {
      "source_model_label": "T2VTurbo [ 32 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "6.22",
      "divisor": 10,
      "source_row": [
        "T2VTurbo [ 32 ]",
        "1.33",
        "0.49",
        "0.43",
        "0.88",
        "0.42",
        "0.96",
        "0.75",
        "0.96",
        "6.22"
      ],
      "conditioning": "text-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "T2VTurbo-v2 (Section 4); 4 steps, cfg 7.5, 16 frames at 16 fps (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "open-sora-1-2",
    "bench": "worldmodelbench-human-total",
    "score": 0.611,
    "source": "https://arxiv.org/html/2502.20694v1#S3.T3",
    "source_id": "video-worldmodelbench",
    "source_group": "video-worldmodelbench:worldmodelbench-human-total",
    "evaluator": "WorldModelBench authors (UC Berkeley / UCSD / MIT / NVIDIA)",
    "by": "independent",
    "date": "2025-02-28",
    "note": "WorldModelBench human annotations, Total 6.11 of 10 (instruction following 0-3 + common sense 0-2 + five physics laws 0-5), averaged over 350 prompts; conditioning text-to-video. Table 3 of arXiv 2502.20694v1. Identity/settings: OpenSora-v1.2 T2V (Section 4); 720p, 4 s, 9:16, 30 steps (Appendix 8.3).",
    "evidence": {
      "source_model_label": "OpenSora-T2V [ 62 ]",
      "table": "Table 3 (S3.T3)",
      "reported_value": "6.11",
      "divisor": 10,
      "source_row": [
        "OpenSora-T2V [ 62 ]",
        "1.71",
        "0.40",
        "0.33",
        "0.89",
        "0.32",
        "0.95",
        "0.60",
        "0.92",
        "6.11"
      ],
      "conditioning": "text-to-video",
      "measurement_origin": "authors' own generations and own large-scale human annotation (67K labels)",
      "identity_evidence": "OpenSora-v1.2 T2V (Section 4); 720p, 4 s, 9:16, 30 steps (Appendix 8.3)",
      "date_basis": "arXiv v1 publication date, not an asserted generation date",
      "source_artifact_sha256": "17b834d6f950359741400033143986f483b43d686e86cf4cf868b53911a487a8"
    }
  },
  {
    "model": "kling-1-6-pro",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 1065,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "independent",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 1,065 (win ratio 61%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: Kling 1.6 'HD'; the report says Kling's outputs were 1080p (Section 4.1), i.e. the Professional/High-Quality mode.",
    "evidence": {
      "source_model_label": "Kling 1.6 HD",
      "table": "Table 3 (S4.T3)",
      "reported_value": "1,065",
      "divisor": 1,
      "source_row": [
        "Kling 1.6 HD",
        "1,065",
        "61%",
        "-"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "Kling 1.6 'HD'; the report says Kling's outputs were 1080p (Section 4.1), i.e. the Professional/High-Quality mode",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "seaweed-7b",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 1047,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "lab",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 1,047 (win ratio 58%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: authors' own model (7B).",
    "evidence": {
      "source_model_label": "Seaweed-7B",
      "table": "Table 3 (S4.T3)",
      "reported_value": "1,047",
      "divisor": 1,
      "source_row": [
        "Seaweed-7B",
        "1,047",
        "58%",
        "7B"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "authors' own model (7B)",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "luma-ray-2",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 1003,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "independent",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 1,003 (win ratio 51%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: named release.",
    "evidence": {
      "source_model_label": "Luma Ray 2",
      "table": "Table 3 (S4.T3)",
      "reported_value": "1,003",
      "divisor": 1,
      "source_row": [
        "Luma Ray 2",
        "1,003",
        "51%",
        "-"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "named release",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "runway-gen-3-alpha",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 1000,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "independent",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 1,000 (win ratio 53%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: named release.",
    "evidence": {
      "source_model_label": "Runway Gen-3 Alpha",
      "table": "Table 3 (S4.T3)",
      "reported_value": "1,000",
      "divisor": 1,
      "source_row": [
        "Runway Gen-3 Alpha",
        "1,000",
        "53%",
        "-"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "named release",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "veo-2",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 992,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "independent",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 992 (win ratio 50%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: named release.",
    "evidence": {
      "source_model_label": "Veo 2.0",
      "table": "Table 3 (S4.T3)",
      "reported_value": "992",
      "divisor": 1,
      "source_row": [
        "Veo 2.0",
        "992",
        "50%",
        "-"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "named release",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "sora",
    "bench": "magicarena-i2v-elo-seaweed",
    "score": 903,
    "source": "https://arxiv.org/html/2504.08685v2#S4.T3",
    "source_id": "video-seaweed-7b",
    "source_group": "video-seaweed-7b:magicarena-i2v-elo-seaweed",
    "evaluator": "ByteDance Seed (Seaweed team) via its MagicArena human Elo evaluation",
    "by": "independent",
    "date": "2025-05-05",
    "note": "Image-to-video Elo 903 (win ratio 36%), Table 3 of arXiv 2504.08685v2: >500 raters, each model in >=7,000 pairwise trials, overall-fidelity side-by-side votes; baselines generated with the models' official APIs as of early March 2025. Identity: Sora via official API/product as of early March 2025.",
    "evidence": {
      "source_model_label": "Sora",
      "table": "Table 3 (S4.T3)",
      "reported_value": "903",
      "divisor": 1,
      "source_row": [
        "Sora",
        "903",
        "36%",
        "-"
      ],
      "source_header": [
        "Name",
        "ELO",
        "Win Ratio",
        "Model Size"
      ],
      "measurement_origin": "authors' own MagicArena human pairwise evaluation reported in the CC BY report",
      "identity_evidence": "Sora via official API/product as of early March 2025",
      "date_basis": "arXiv v2 date; generations as of early March 2025",
      "source_artifact_sha256": "d414cda3e313099eebbd5543caad94ac2a5a78c7ae5f324cdec74517a7bce06b"
    }
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "videoverse-overall",
    "score": 0.556,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 894 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: CogVideoX1.5 5B variant (caption), (S) = 5 s short-video setting; the 10 s (L) setting of the same checkpoint (893) is not imported.",
    "evidence": {
      "source_model_label": "CogVideoX1.5 (S)",
      "table": "Table 1 (S3.T1)",
      "reported_value": "894",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "CogVideoX1.5 5B variant (caption), (S) = 5 s short-video setting; the 10 s (L) setting of the same checkpoint (893) is not imported",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "hunyuanvideo",
    "bench": "videoverse-overall",
    "score": 0.5585,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 898 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: Appendix D.1: accessed from github.com/Tencent-Hunyuan/HunyuanVideo.",
    "evidence": {
      "source_model_label": "Hunyuan",
      "table": "Table 1 (S3.T1)",
      "reported_value": "898",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "Appendix D.1: accessed from github.com/Tencent-Hunyuan/HunyuanVideo",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "open-sora-2-0",
    "bench": "videoverse-overall",
    "score": 0.615,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 989 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: Appendix D.1: accessed from github.com/hpcaitech/Open-Sora.",
    "evidence": {
      "source_model_label": "OpenSora2.0",
      "table": "Table 1 (S3.T1)",
      "reported_value": "989",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "Appendix D.1: accessed from github.com/hpcaitech/Open-Sora",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "videoverse-overall",
    "score": 0.6748,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 1085 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: Appendix D.1: Wan2.2-A14B with checkpoints from huggingface.co/Wan-AI/Wan2.2-T2V-A14B.",
    "evidence": {
      "source_model_label": "Wan2.2-A14B",
      "table": "Table 1 (S3.T1)",
      "reported_value": "1085",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "Appendix D.1: Wan2.2-A14B with checkpoints from huggingface.co/Wan-AI/Wan2.2-T2V-A14B",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "veo-3-fast",
    "bench": "videoverse-overall",
    "score": 0.8035,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 1292 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: Appendix D.2: 'Due to the high cost of Veo-3 and Sora-2, we employ Veo-3-fast and Sora-2 standard versions'.",
    "evidence": {
      "source_model_label": "Veo-3",
      "table": "Table 1 (S3.T1)",
      "reported_value": "1292",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "Appendix D.2: 'Due to the high cost of Veo-3 and Sora-2, we employ Veo-3-fast and Sora-2 standard versions'",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "sora-2",
    "bench": "videoverse-overall",
    "score": 0.8078,
    "source": "https://arxiv.org/html/2510.08398v4#S3.T1",
    "source_id": "video-videoverse",
    "source_group": "video-videoverse:videoverse-overall",
    "evaluator": "VideoVerse authors (Sun Yat-sen University / Hong Kong Polytechnic University)",
    "by": "independent",
    "date": "2026-05-15",
    "note": "VideoVerse Overall 1299 (Gemini 2.5 Pro judge) divided by 1608 = 793 binary questions + 815 events (Table S8; Overall = basic + world-model incl. event-following LCS, Table 2). Table 1 of arXiv 2510.08398v4. Identity: Appendix D.2: Sora-2 standard via api.openai.com/v1/videos; 4 videos not generated (security review).",
    "evidence": {
      "source_model_label": "Sora-2*",
      "table": "Table 1 (S3.T1)",
      "reported_value": "1299",
      "divisor": 1608,
      "divisor_basis": "Table S8: 'VideoVerse 793+815 Events'; Table S3: 478 basic + 315 world-model binary questions, 815 events; Table 2 shows Overall = Basic + World Model w/ EF",
      "measurement_origin": "authors' own generations (open models on A800s, closed via official APIs) and own VLM evaluation; user study Table 3",
      "identity_evidence": "Appendix D.2: Sora-2 standard via api.openai.com/v1/videos; 4 videos not generated (security review)",
      "date_basis": "arXiv v4 date (15 May 2026)",
      "source_artifact_sha256": "c503e2524ba788da70ca3357756caf705a34676b2ea369e7dd60968ebdb91150"
    }
  },
  {
    "model": "runway-gen-2",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.524,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.62 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Runway Gen-2; already evaluated in v1 (26 Jun 2024); the authors' repository leaderboard labels it 'Gen-2 (20240610)'.",
    "evidence": {
      "source_model_label": "Gen-2 [ 58 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.62",
      "divisor": 5,
      "source_row": [
        "Gen-2 [ 58 ]",
        "Runway",
        "U-Net",
        "Close-Source",
        "218.99",
        "2.400",
        "0.373",
        "125.25",
        "2.62"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Runway Gen-2; already evaluated in v1 (26 Jun 2024); the authors' repository leaderboard labels it 'Gen-2 (20240610)'",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "pika-1-0",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.496,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.48 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: printed 'Pika-2.0' in v1 (26 Jun 2024) and v2, but Pika 2.0 was released in Dec 2024 and Pika 1.5 in Oct 2024; the authors' repository leaderboard (results_ChronoMagic-Bench-150.csv) labels the same row 'Pika-1.0 (20240610)'.",
    "evidence": {
      "source_model_label": "Pika-2.0 [ 34 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.48",
      "divisor": 5,
      "source_row": [
        "Pika-2.0 [ 34 ]",
        "PikaLab",
        "U-Net",
        "Close-Source",
        "223.05",
        "2.317",
        "0.347",
        "75.98",
        "2.48"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "printed 'Pika-2.0' in v1 (26 Jun 2024) and v2, but Pika 2.0 was released in Dec 2024 and Pika 1.5 in Oct 2024; the authors' repository leaderboard (results_ChronoMagic-Bench-150.csv) labels the same row 'Pika-1.0 (20240610)'",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "luma-dream-machine-1-0",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.622,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 3.11 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Luma Dream Machine as reported in v1 (26 Jun 2024), i.e. the June 2024 launch model; repository leaderboard label 'Dream Machine (20240610)'.",
    "evidence": {
      "source_model_label": "Dream Machine [ 46 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "3.11",
      "divisor": 5,
      "source_row": [
        "Dream Machine [ 46 ]",
        "LUMA",
        "DiT",
        "Close-Source",
        "214.91",
        "2.387",
        "0.474",
        "95.97",
        "3.11"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Luma Dream Machine as reported in v1 (26 Jun 2024), i.e. the June 2024 launch model; repository leaderboard label 'Dream Machine (20240610)'",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "kling-1-0",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.548,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.74 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Kuaishou Kling (KeLing) as reported in v1 (26 Jun 2024), i.e. the June 2024 Kling 1.0 launch; repository leaderboard label 'KeLing (20240610)'.",
    "evidence": {
      "source_model_label": "KeLing [ 33 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.74",
      "divisor": 5,
      "source_row": [
        "KeLing [ 33 ]",
        "Kwai",
        "DiT",
        "Close-Source",
        "202.32",
        "2.517",
        "0.369",
        "74.20",
        "2.74"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Kuaishou Kling (KeLing) as reported in v1 (26 Jun 2024), i.e. the June 2024 Kling 1.0 launch; repository leaderboard label 'KeLing (20240610)'",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.602,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 3.01 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: official ModelScopeT2V HuggingFace model, 256x256, 16 frames at 8 fps.",
    "evidence": {
      "source_model_label": "ModelScopeT2V [ 68 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "3.01",
      "divisor": 5,
      "source_row": [
        "ModelScopeT2V [ 68 ]",
        "Arxiv’23",
        "U-Net",
        "Open-Source",
        "230.74",
        "2.783",
        "0.409",
        "61.01",
        "3.01"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: official ModelScopeT2V HuggingFace model, 256x256, 16 frames at 8 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "zeroscope-v2-576w",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.458,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.29 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: ZeroScope_v2_576w, 576x320, 24 frames at 8 fps.",
    "evidence": {
      "source_model_label": "ZeroScope [ 64 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.29",
      "divisor": 5,
      "source_row": [
        "ZeroScope [ 64 ]",
        "CVPR’23",
        "U-Net",
        "Open-Source",
        "260.61",
        "2.232",
        "0.403",
        "94.67",
        "2.29"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: ZeroScope_v2_576w, 576x320, 24 frames at 8 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "lavie",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.5,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.50 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: official LaVie base model without interpolation/super-resolution, 512x320, 16 frames.",
    "evidence": {
      "source_model_label": "LaVie [ 71 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.50",
      "divisor": 5,
      "source_row": [
        "LaVie [ 71 ]",
        "Arxiv’23",
        "U-Net",
        "Open-Source",
        "210.39",
        "2.714",
        "0.350",
        "81.32",
        "2.50"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: official LaVie base model without interpolation/super-resolution, 512x320, 16 frames",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "videocrafter2",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.574,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.87 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: official VideoCrafter2, 512x320, 20 frames at 10 fps.",
    "evidence": {
      "source_model_label": "VideoCrafter2 [ 11 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.87",
      "divisor": 5,
      "source_row": [
        "VideoCrafter2 [ 11 ]",
        "CVPR’23",
        "U-Net",
        "Open-Source",
        "214.06",
        "2.763",
        "0.437",
        "75.90",
        "2.87"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: official VideoCrafter2, 512x320, 20 frames at 10 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "open-sora-1-1",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.514,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.57 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: Open-Sora 1.1 stage-3 checkpoint, 512x512, 48 frames at 24 fps.",
    "evidence": {
      "source_model_label": "OpenSora 1.1 [ 90 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.57",
      "divisor": 5,
      "source_row": [
        "OpenSora 1.1 [ 90 ]",
        "Github’24",
        "DiT",
        "Open-Source",
        "241.09",
        "2.676",
        "0.448",
        "75.94",
        "2.57"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: Open-Sora 1.1 stage-3 checkpoint, 512x512, 48 frames at 24 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "open-sora-1-2",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.5,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 2.50 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: Open-Sora 1.2, 1280x720, 96 frames at 24 fps.",
    "evidence": {
      "source_model_label": "OpenSora 1.2 [ 90 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "2.50",
      "divisor": 5,
      "source_row": [
        "OpenSora 1.2 [ 90 ]",
        "Github’24",
        "DiT",
        "Open-Source",
        "210.93",
        "2.681",
        "0.383",
        "51.87",
        "2.50"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: Open-Sora 1.2, 1280x720, 96 frames at 24 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "cogvideox-2b",
    "bench": "chronomagic-150-gpt4o-mtscore",
    "score": 0.618,
    "source": "https://arxiv.org/html/2406.18522v2#S5.T4",
    "source_id": "video-chronomagic-bench",
    "source_group": "video-chronomagic-bench:chronomagic-150-gpt4o-mtscore",
    "evaluator": "ChronoMagic-Bench authors (Peking University / University of Rochester)",
    "by": "independent",
    "date": "2024-10-01",
    "note": "ChronoMagic-Bench-150 GPT4o-MTScore 3.09 on a 1-5 scale (GPT-4o rates the metamorphic amplitude of uniformly sampled frames), stored as value/5. Table 4 of arXiv 2406.18522v2 (closed models and open models on the same 150 time-lapse prompts, three videos per prompt). Identity: Appendix: official CogVideoX (2B per Tables 3-4), 720x480, 48 frames at 8 fps.",
    "evidence": {
      "source_model_label": "CogVideoX-2B [ 81 ]",
      "table": "Table 4 (S5.T4)",
      "reported_value": "3.09",
      "divisor": 5,
      "source_row": [
        "CogVideoX-2B [ 81 ]",
        "Arxiv’24",
        "DiT",
        "Open-Source",
        "195.52",
        "3.240",
        "0.472",
        "38.64",
        "3.09"
      ],
      "source_header": [
        "Method",
        "Venue",
        "Backbone",
        "Status",
        "UMT-FVD ↓ \\downarrow",
        "UMTScore ↑ \\uparrow",
        "MTScore ↑ \\uparrow",
        "CHScore ↑ \\uparrow",
        "GPT4o-MTScore ↑ \\uparrow"
      ],
      "measurement_origin": "authors' own generations (open models run locally with official code; closed models via their products) and own GPT-4o scoring",
      "identity_evidence": "Appendix: official CogVideoX (2B per Tables 3-4), 720x480, 48 frames at 8 fps",
      "date_basis": "arXiv v2 date (1 Oct 2024); closed-model rows already present in v1 (26 Jun 2024)",
      "source_artifact_sha256": "3343c2a324880815f9639cd186e3791acfae56ce39e1e1b8bc98f8baf4c77e2e"
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "lambada-openai",
    "score": 0.516,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Ada‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "51.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "winogrande",
    "score": 0.529,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Ada‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "52.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "hellaswag",
    "score": 0.434,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Ada‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "43.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "piqa",
    "score": 0.705,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Ada‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "70.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "lambada-openai",
    "score": 0.5121,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-2 1.5B', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "51.21%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "winogrande",
    "score": 0.594,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-2 1.5B', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "Winogrande ↑",
      "raw_value": "59.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "hellaswag",
    "score": 0.509,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-2 1.5B', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "Hellaswag ↑",
      "raw_value": "50.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "piqa",
    "score": 0.708,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-2 1.5B', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "PIQA ↑",
      "raw_value": "70.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "lambada-openai",
    "score": 0.572,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 1.3B‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "57.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "winogrande",
    "score": 0.55,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 1.3B‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "55.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "hellaswag",
    "score": 0.489,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 1.3B‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "48.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "piqa",
    "score": 0.711,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 1.3B‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "71.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "lambada-openai",
    "score": 0.622,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 2.7B‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 2.7B&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "62.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "winogrande",
    "score": 0.565,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 2.7B‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 2.7B&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "56.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "hellaswag",
    "score": 0.558,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 2.7B‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 2.7B&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "55.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "piqa",
    "score": 0.73,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-Neo 2.7B‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 2.7B&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "73.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "lambada-openai",
    "score": 0.624,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Babbage‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Babbage&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "62.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "winogrande",
    "score": 0.59,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Babbage‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Babbage&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "59.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "hellaswag",
    "score": 0.545,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Babbage‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Babbage&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "54.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "piqa",
    "score": 0.755,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Babbage‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Babbage&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "75.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "lambada-openai",
    "score": 0.697,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-J 6B‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-J 6B&ddagger;**",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "69.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "winogrande",
    "score": 0.653,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-J 6B‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-J 6B&ddagger;**",
      "column_label": "Winogrande ↑",
      "raw_value": "65.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "hellaswag",
    "score": 0.661,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-J 6B‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-J 6B&ddagger;**",
      "column_label": "Hellaswag ↑",
      "raw_value": "66.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "piqa",
    "score": 0.765,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "lab",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-J 6B‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-J 6B&ddagger;**",
      "column_label": "PIQA ↑",
      "raw_value": "76.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "lambada-openai",
    "score": 0.693,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Curie‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Curie&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "69.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "winogrande",
    "score": 0.656,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Curie‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Curie&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "65.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "hellaswag",
    "score": 0.685,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Curie‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Curie&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "68.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "piqa",
    "score": 0.779,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Curie‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Curie&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "77.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "lambada-openai",
    "score": 0.75,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Davinci‡', column 'LAMBADA Acc ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). LAMBADA is the harness 'lambada' task (OpenAI detokenised test file). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Davinci&ddagger;",
      "column_label": "LAMBADA Acc ↑",
      "raw_value": "75%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "winogrande",
    "score": 0.72,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Davinci‡', column 'Winogrande ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Davinci&ddagger;",
      "column_label": "Winogrande ↑",
      "raw_value": "72%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "hellaswag",
    "score": 0.78,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Davinci‡', column 'Hellaswag ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Davinci&ddagger;",
      "column_label": "Hellaswag ↑",
      "raw_value": "78%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "piqa",
    "score": 0.8,
    "source": "https://huggingface.co/EleutherAI/gpt-j-6b/blob/47e169305d2e8376be1d31e765533382721b2cc1/README.md",
    "source_id": "eleutherai-gpt-j-6b-card",
    "by": "independent",
    "date": "2021-08-06",
    "mode": "base",
    "note": "EleutherAI GPT-J-6B model card, 'Evaluation results' table, row 'GPT-3 Davinci‡', column 'PIQA ↑'. Zero-shot, EleutherAI lm-evaluation-harness (2021; API access for OpenAI models, released weights otherwise); rows marked * (numbers reported by other authors) are not used. Original OpenAI API base model evaluated in 2021 (before its 2024 retirement). Table first appeared in the card on 2021-08-06 (commit d3d295606169); cited at pinned revision 47e1693.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Davinci&ddagger;",
      "column_label": "PIQA ↑",
      "raw_value": "80%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "mathqa",
    "score": 0.2405,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "lab",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-Neo 1.3B', column 'MathQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B",
      "column_label": "MathQA",
      "raw_value": "24.05%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "pubmedqa",
    "score": 0.544,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "lab",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-Neo 1.3B', column 'PubMedQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-Neo 1.3B",
      "column_label": "PubMedQA",
      "raw_value": "54.40%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "mathqa",
    "score": 0.2364,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "independent",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-2 1.5B', column 'MathQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "MathQA",
      "raw_value": "23.64%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "pubmedqa",
    "score": 0.5833,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "independent",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-2 1.5B', column 'PubMedQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-2 1.5B",
      "column_label": "PubMedQA",
      "raw_value": "58.33%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "mathqa",
    "score": 0.2472,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "lab",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-Neo 2.7B', column 'MathQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-Neo 2.7B**",
      "column_label": "MathQA",
      "raw_value": "24.72%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "pubmedqa",
    "score": 0.5754,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "lab",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-Neo 2.7B', column 'PubMedQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "**GPT-Neo 2.7B**",
      "column_label": "PubMedQA",
      "raw_value": "57.54%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "mathqa",
    "score": 0.2429,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "independent",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-3 Ada', column 'MathQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). OpenAI API base model 'ada' as of 2021. Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada",
      "column_label": "MathQA",
      "raw_value": "24.29%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "pubmedqa",
    "score": 0.528,
    "source": "https://huggingface.co/EleutherAI/gpt-neo-2.7B/blob/e24fa291132763e59f4a5422741b424fb5d59056/README.md",
    "source_id": "eleutherai-gpt-neo-2-7b-card",
    "by": "independent",
    "date": "2021-04-12",
    "mode": "base",
    "note": "EleutherAI GPT-Neo 2.7B model card, 'Eval results > Physical and Scientific Reasoning' table, row 'GPT-3 Ada', column 'PubMedQA'. Zero-shot, EleutherAI lm-evaluation-harness (card: 'All evaluations were done using our evaluation harness'). OpenAI API base model 'ada' as of 2021. Only MathQA and PubMedQA are taken from this card; its LAMBADA/Winogrande/PIQA/HellaSwag cells repeat or overlap the GPT-J card's runs. Table first appeared in the card on 2021-04-12 (commit 3e1c92ca77a1); cited at pinned revision e24fa29.",
    "evidence": {
      "kind": "markdown-table",
      "row_label": "GPT-3 Ada",
      "column_label": "PubMedQA",
      "raw_value": "52.80%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "lambada-openai",
    "score": 0.683,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-J 6B",
      "raw_value": "0.683±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "lambada-openai",
    "score": 0.72,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.720±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "lambada-openai",
    "score": 0.515,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.515±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "lambada-openai",
    "score": 0.625,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.625±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "lambada-openai",
    "score": 0.693,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.693±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "lambada-openai",
    "score": 0.752,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'LAMBADA', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. LAMBADA is the harness v0 'lambada' task (OpenAI detokenised file). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "LAMBADA",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.752±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "hellaswag",
    "score": 0.518,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-J 6B",
      "raw_value": "0.518±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "hellaswag",
    "score": 0.535,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.535±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "hellaswag",
    "score": 0.359,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.359±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "hellaswag",
    "score": 0.429,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.429±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "hellaswag",
    "score": 0.505,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.505±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "hellaswag",
    "score": 0.592,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'HellaSwag', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "HellaSwag",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.592±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "winogrande",
    "score": 0.64,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-J 6B",
      "raw_value": "0.640±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "winogrande",
    "score": 0.661,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.661±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "winogrande",
    "score": 0.528,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.528±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "winogrande",
    "score": 0.594,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.594±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "winogrande",
    "score": 0.649,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.649±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "winogrande",
    "score": 0.699,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'Winogrande', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "Winogrande",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.699±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "sciq",
    "score": 0.91,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-J 6B",
      "raw_value": "0.910±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "sciq",
    "score": 0.928,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.928±0.008",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "sciq",
    "score": 0.843,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.843±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "sciq",
    "score": 0.866,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.866±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "sciq",
    "score": 0.918,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.918±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "sciq",
    "score": 0.949,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'SciQ', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "SciQ",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.949±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "piqa",
    "score": 0.752,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-J 6B",
      "raw_value": "0.752±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "piqa",
    "score": 0.779,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.779±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "piqa",
    "score": 0.69,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.690±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "piqa",
    "score": 0.745,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.745±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "piqa",
    "score": 0.767,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.767±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "piqa",
    "score": 0.791,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'PIQA', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "PIQA",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.791±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "triviaqa",
    "score": 0.17,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-J 6B",
      "raw_value": "0.170±0.004",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "triviaqa",
    "score": 0.259,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.259±0.004",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "triviaqa",
    "score": 0.05,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.050±0.002",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "triviaqa",
    "score": 0.115,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.115±0.003",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "triviaqa",
    "score": 0.196,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.196±0.004",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "triviaqa",
    "score": 0.409,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'TriviaQA', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. TriviaQA as implemented in harness v0 (zero-shot, accuracy). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "TriviaQA",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.409±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "arc-easy",
    "score": 0.67,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-J 6B",
      "raw_value": "0.670±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "arc-easy",
    "score": 0.723,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.723±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "arc-easy",
    "score": 0.514,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.514±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "arc-easy",
    "score": 0.598,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.598±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "arc-easy",
    "score": 0.682,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.682±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "arc-easy",
    "score": 0.762,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Easy)', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Easy)",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.762±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "arc-challenge",
    "score": 0.34,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-J 6B",
      "raw_value": "0.340±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "arc-challenge",
    "score": 0.38,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.380±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "arc-challenge",
    "score": 0.225,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.225±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "arc-challenge",
    "score": 0.275,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.275±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "arc-challenge",
    "score": 0.334,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.334±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "arc-challenge",
    "score": 0.435,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'ARC (Challenge)', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "ARC (Challenge)",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.435±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-j-6b",
    "bench": "openbookqa",
    "score": 0.288,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-J 6B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. GPT-J-6B is an EleutherAI release (its author Ben Wang co-wrote this paper), so it is marked lab. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-J 6B",
      "raw_value": "0.288±0.020",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neox-20b",
    "bench": "openbookqa",
    "score": 0.29,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "lab",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-NeoX 20B' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-NeoX 20B",
      "raw_value": "0.290±0.020",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-ada",
    "bench": "openbookqa",
    "score": 0.172,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-3 Ada' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-3 Ada",
      "raw_value": "0.172±0.017",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-babbage",
    "bench": "openbookqa",
    "score": 0.224,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-3 Babbage' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-3 Babbage",
      "raw_value": "0.224±0.019",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-curie",
    "bench": "openbookqa",
    "score": 0.29,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-3 Curie' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-3 Curie",
      "raw_value": "0.290±0.020",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-3-davinci",
    "bench": "openbookqa",
    "score": 0.336,
    "source": "https://arxiv.org/html/2204.06745v1",
    "source_id": "paper-2204.06745v1",
    "by": "independent",
    "date": "2022-04-14",
    "mode": "base",
    "note": "GPT-NeoX-20B paper (arXiv:2204.06745v1) Appendix D, Table 2 'Zero-Shot Results on Natural Language Understanding Tasks', row 'OpenBookQA', column 'GPT-3 DaVinci' (value before ±). EleutherAI harness, task version 0, zero-shot; GPT-3 models evaluated through the OpenAI API in early 2022. ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Appendix D)",
      "row_label": "OpenBookQA",
      "column_label": "GPT-3 DaVinci",
      "raw_value": "0.336±0.021",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "lambada-openai",
    "score": 0.185,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "70M",
      "raw_value": "0.185±0.005",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "lambada-openai",
    "score": 0.328,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "160M",
      "raw_value": "0.328±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "lambada-openai",
    "score": 0.516,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "410M",
      "raw_value": "0.516±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "lambada-openai",
    "score": 0.562,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "1B",
      "raw_value": "0.562±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "piqa",
    "score": 0.595,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "70M",
      "raw_value": "0.595±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "piqa",
    "score": 0.627,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "160M",
      "raw_value": "0.627±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "piqa",
    "score": 0.668,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "410M",
      "raw_value": "0.668±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "piqa",
    "score": 0.707,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "1B",
      "raw_value": "0.707±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "winogrande",
    "score": 0.528,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "70M",
      "raw_value": "0.528±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "winogrande",
    "score": 0.531,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "160M",
      "raw_value": "0.531±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "winogrande",
    "score": 0.537,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "410M",
      "raw_value": "0.537±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "winogrande",
    "score": 0.537,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "1B",
      "raw_value": "0.537±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "arc-easy",
    "score": 0.374,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "70M",
      "raw_value": "0.374±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "arc-easy",
    "score": 0.435,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "160M",
      "raw_value": "0.435±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "arc-easy",
    "score": 0.521,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "410M",
      "raw_value": "0.521±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "arc-easy",
    "score": 0.569,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "1B",
      "raw_value": "0.569±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "arc-challenge",
    "score": 0.181,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "70M",
      "raw_value": "0.181±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "arc-challenge",
    "score": 0.188,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "160M",
      "raw_value": "0.188±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "arc-challenge",
    "score": 0.213,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "410M",
      "raw_value": "0.213±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "arc-challenge",
    "score": 0.244,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "1B",
      "raw_value": "0.244±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-70m",
    "bench": "sciq",
    "score": 0.601,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '70M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "70M",
      "raw_value": "0.601±0.015",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-160m",
    "bench": "sciq",
    "score": 0.741,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '160M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "160M",
      "raw_value": "0.741±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-410m",
    "bench": "sciq",
    "score": 0.811,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '410M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "410M",
      "raw_value": "0.811±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "sciq",
    "score": 0.84,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "1B",
      "raw_value": "0.840±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "lambada-openai",
    "score": 0.616,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "1.4B",
      "raw_value": "0.616±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "lambada-openai",
    "score": 0.647,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "2.8B",
      "raw_value": "0.647±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "lambada-openai",
    "score": 0.673,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "6.9B",
      "raw_value": "0.673±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "lambada-openai",
    "score": 0.705,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'Lambada (OpenAI)', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "Lambada (OpenAI)",
      "column_label": "12B",
      "raw_value": "0.705±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "piqa",
    "score": 0.711,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "1.4B",
      "raw_value": "0.711±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "piqa",
    "score": 0.739,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "2.8B",
      "raw_value": "0.739±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "piqa",
    "score": 0.752,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "6.9B",
      "raw_value": "0.752±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "piqa",
    "score": 0.76,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'PIQA', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "PIQA",
      "column_label": "12B",
      "raw_value": "0.760±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "winogrande",
    "score": 0.573,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "1.4B",
      "raw_value": "0.573±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "winogrande",
    "score": 0.594,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "2.8B",
      "raw_value": "0.594±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "winogrande",
    "score": 0.609,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "6.9B",
      "raw_value": "0.609±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "winogrande",
    "score": 0.639,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'WinoGrande', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "WinoGrande",
      "column_label": "12B",
      "raw_value": "0.639±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "arc-easy",
    "score": 0.606,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "1.4B",
      "raw_value": "0.606±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "arc-easy",
    "score": 0.644,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "2.8B",
      "raw_value": "0.644±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-easy",
    "score": 0.673,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "6.9B",
      "raw_value": "0.673±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "arc-easy",
    "score": 0.702,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Easy', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Easy",
      "column_label": "12B",
      "raw_value": "0.702±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "arc-challenge",
    "score": 0.26,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "1.4B",
      "raw_value": "0.260±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "arc-challenge",
    "score": 0.295,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "2.8B",
      "raw_value": "0.295±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-challenge",
    "score": 0.313,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "6.9B",
      "raw_value": "0.313±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "arc-challenge",
    "score": 0.318,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'ARC - Challenge', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "ARC - Challenge",
      "column_label": "12B",
      "raw_value": "0.318±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1-4b",
    "bench": "sciq",
    "score": 0.865,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '1.4B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "1.4B",
      "raw_value": "0.865±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-2-8b",
    "bench": "sciq",
    "score": 0.882,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '2.8B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "2.8B",
      "raw_value": "0.882±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "sciq",
    "score": 0.897,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '6.9B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "6.9B",
      "raw_value": "0.897±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-12b",
    "bench": "sciq",
    "score": 0.902,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "lab",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 7, row 'SciQ', column '12B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. Final checkpoint of the standard (non-deduplicated) Pythia model. ",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "SciQ",
      "column_label": "12B",
      "raw_value": "0.902±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "lambada-openai",
    "score": 0.341,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'Lambada (OpenAI)', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Lambada (OpenAI)",
      "column_label": "560M",
      "raw_value": "0.341±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "lambada-openai",
    "score": 0.426,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'Lambada (OpenAI)', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Lambada (OpenAI)",
      "column_label": "1.1B",
      "raw_value": "0.426±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "lambada-openai",
    "score": 0.462,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'Lambada (OpenAI)', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Lambada (OpenAI)",
      "column_label": "1.7B",
      "raw_value": "0.462±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "lambada-openai",
    "score": 0.518,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'Lambada (OpenAI)', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Lambada (OpenAI)",
      "column_label": "3B",
      "raw_value": "0.518±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "piqa",
    "score": 0.637,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'PIQA', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "PIQA",
      "column_label": "560M",
      "raw_value": "0.637±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "piqa",
    "score": 0.672,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'PIQA', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "PIQA",
      "column_label": "1.1B",
      "raw_value": "0.672±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "piqa",
    "score": 0.686,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'PIQA', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "PIQA",
      "column_label": "1.7B",
      "raw_value": "0.686±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "piqa",
    "score": 0.708,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'PIQA', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "PIQA",
      "column_label": "3B",
      "raw_value": "0.708±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "winogrande",
    "score": 0.504,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'WinoGrande', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "WinoGrande",
      "column_label": "560M",
      "raw_value": "0.504±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "winogrande",
    "score": 0.547,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'WinoGrande', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "WinoGrande",
      "column_label": "1.1B",
      "raw_value": "0.547±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "winogrande",
    "score": 0.572,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'WinoGrande', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "WinoGrande",
      "column_label": "1.7B",
      "raw_value": "0.572±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "winogrande",
    "score": 0.586,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'WinoGrande', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "WinoGrande",
      "column_label": "3B",
      "raw_value": "0.586±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "arc-easy",
    "score": 0.476,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC - Easy', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC - Easy",
      "column_label": "560M",
      "raw_value": "0.476±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "arc-easy",
    "score": 0.515,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC - Easy', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC - Easy",
      "column_label": "1.1B",
      "raw_value": "0.515±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "arc-easy",
    "score": 0.562,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC - Easy', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC - Easy",
      "column_label": "1.7B",
      "raw_value": "0.562±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "arc-easy",
    "score": 0.594,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC - Easy', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC - Easy",
      "column_label": "3B",
      "raw_value": "0.594±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "arc-challenge",
    "score": 0.221,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC — Challenge', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC — Challenge",
      "column_label": "560M",
      "raw_value": "0.221±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "arc-challenge",
    "score": 0.236,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC — Challenge', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC — Challenge",
      "column_label": "1.1B",
      "raw_value": "0.236±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "arc-challenge",
    "score": 0.238,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC — Challenge', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC — Challenge",
      "column_label": "1.7B",
      "raw_value": "0.238±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "arc-challenge",
    "score": 0.28,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'ARC — Challenge', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "ARC — Challenge",
      "column_label": "3B",
      "raw_value": "0.280±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-560m",
    "bench": "sciq",
    "score": 0.804,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'SciQ', column '560M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "SciQ",
      "column_label": "560M",
      "raw_value": "0.804±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "sciq",
    "score": 0.833,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'SciQ', column '1.1B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "SciQ",
      "column_label": "1.1B",
      "raw_value": "0.833±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "sciq",
    "score": 0.851,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'SciQ', column '1.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "SciQ",
      "column_label": "1.7B",
      "raw_value": "0.851±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "bloom-3b",
    "bench": "sciq",
    "score": 0.891,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 15, row 'SciQ', column '3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "SciQ",
      "column_label": "3B",
      "raw_value": "0.891±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "lambada-openai",
    "score": 0.379,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "125M",
      "raw_value": "0.379±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "lambada-openai",
    "score": 0.452,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "350M",
      "raw_value": "0.452±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "lambada-openai",
    "score": 0.579,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "1.3B",
      "raw_value": "0.579±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "lambada-openai",
    "score": 0.636,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "2.7B",
      "raw_value": "0.636±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "piqa",
    "score": 0.63,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "125M",
      "raw_value": "0.630±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "piqa",
    "score": 0.644,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "350M",
      "raw_value": "0.644±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "piqa",
    "score": 0.717,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "1.3B",
      "raw_value": "0.717±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "piqa",
    "score": 0.739,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "2.7B",
      "raw_value": "0.739±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "winogrande",
    "score": 0.503,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "125M",
      "raw_value": "0.503±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "winogrande",
    "score": 0.523,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "350M",
      "raw_value": "0.523±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "winogrande",
    "score": 0.597,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "1.3B",
      "raw_value": "0.597±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "winogrande",
    "score": 0.61,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "2.7B",
      "raw_value": "0.610±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "arc-easy",
    "score": 0.435,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "125M",
      "raw_value": "0.435±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "arc-easy",
    "score": 0.44,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "350M",
      "raw_value": "0.440±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "arc-easy",
    "score": 0.57,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "1.3B",
      "raw_value": "0.570±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "arc-easy",
    "score": 0.608,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "2.7B",
      "raw_value": "0.608±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "arc-challenge",
    "score": 0.189,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "125M",
      "raw_value": "0.189±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "arc-challenge",
    "score": 0.207,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "350M",
      "raw_value": "0.207±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "arc-challenge",
    "score": 0.231,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "1.3B",
      "raw_value": "0.231±0.012",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "arc-challenge",
    "score": 0.268,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "2.7B",
      "raw_value": "0.268±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-125m",
    "bench": "sciq",
    "score": 0.751,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '125M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "125M",
      "raw_value": "0.751±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-350m",
    "bench": "sciq",
    "score": 0.748,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '350M' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "350M",
      "raw_value": "0.748±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "sciq",
    "score": 0.845,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '1.3B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "1.3B",
      "raw_value": "0.845±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-2-7b",
    "bench": "sciq",
    "score": 0.858,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '2.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "2.7B",
      "raw_value": "0.858±0.011",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "lambada-openai",
    "score": 0.677,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "6.7B",
      "raw_value": "0.677±0.007",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "lambada-openai",
    "score": 0.686,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "13B",
      "raw_value": "0.686±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "lambada-openai",
    "score": 0.715,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "30B",
      "raw_value": "0.715±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "lambada-openai",
    "score": 0.739,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'Lambada (OpenAI)', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. LAMBADA (OpenAI) = OpenAI detokenised test file. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "Lambada (OpenAI)",
      "column_label": "66B",
      "raw_value": "0.739±0.006",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "piqa",
    "score": 0.763,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "6.7B",
      "raw_value": "0.763±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "piqa",
    "score": 0.76,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "13B",
      "raw_value": "0.760±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "piqa",
    "score": 0.776,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "30B",
      "raw_value": "0.776±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "piqa",
    "score": 0.788,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'PIQA', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "PIQA",
      "column_label": "66B",
      "raw_value": "0.788±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "winogrande",
    "score": 0.653,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "6.7B",
      "raw_value": "0.653±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "winogrande",
    "score": 0.652,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "13B",
      "raw_value": "0.652±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "winogrande",
    "score": 0.682,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "30B",
      "raw_value": "0.682±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "winogrande",
    "score": 0.687,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'WinoGrande', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "WinoGrande",
      "column_label": "66B",
      "raw_value": "0.687±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "arc-easy",
    "score": 0.656,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "6.7B",
      "raw_value": "0.656±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "arc-easy",
    "score": 0.671,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "13B",
      "raw_value": "0.671±0.010",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "arc-easy",
    "score": 0.7,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "30B",
      "raw_value": "0.700±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "arc-easy",
    "score": 0.717,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC - Easy', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC - Easy",
      "column_label": "66B",
      "raw_value": "0.717±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "arc-challenge",
    "score": 0.305,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "6.7B",
      "raw_value": "0.305±0.013",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "arc-challenge",
    "score": 0.329,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "13B",
      "raw_value": "0.329±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "arc-challenge",
    "score": 0.346,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "30B",
      "raw_value": "0.346±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "arc-challenge",
    "score": 0.372,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'ARC — Challenge', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "ARC — Challenge",
      "column_label": "66B",
      "raw_value": "0.372±0.014",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-6-7b",
    "bench": "sciq",
    "score": 0.901,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '6.7B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "6.7B",
      "raw_value": "0.901±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-13b",
    "bench": "sciq",
    "score": 0.908,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '13B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "13B",
      "raw_value": "0.908±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-30b",
    "bench": "sciq",
    "score": 0.911,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '30B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "30B",
      "raw_value": "0.911±0.009",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-66b",
    "bench": "sciq",
    "score": 0.926,
    "source": "https://arxiv.org/html/2304.01373v2",
    "source_id": "paper-2304.01373v2",
    "by": "independent",
    "date": "2023-05-31",
    "mode": "base",
    "note": "Pythia paper (arXiv:2304.01373v2) Appendix G, Table 16, row 'SciQ', column '66B' (value before ±). EleutherAI LM Evaluation Harness, zero-shot; the authors ran all evaluations themselves. ",
    "evidence": {
      "kind": "table",
      "table": "Table 16",
      "row_label": "SciQ",
      "column_label": "66B",
      "raw_value": "0.926±0.008",
      "scale_divisor": 1
    }
  },
  {
    "model": "pythia-1b",
    "bench": "arc-challenge",
    "score": 0.331,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "arc challenge",
      "raw_value": "33.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "arc-easy",
    "score": 0.502,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "arc easy",
      "raw_value": "50.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "boolq",
    "score": 0.618,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "boolq",
      "raw_value": "61.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "hellaswag",
    "score": 0.447,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "hellaswag",
      "raw_value": "44.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "openbookqa",
    "score": 0.378,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "openbookqa",
      "raw_value": "37.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "piqa",
    "score": 0.691,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "piqa",
      "raw_value": "69.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "sciq",
    "score": 0.86,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "sciq",
      "raw_value": "86.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-1b",
    "bench": "winogrande",
    "score": 0.533,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 1B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 1B",
      "column_label": "winogrande",
      "raw_value": "53.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "arc-challenge",
    "score": 0.345,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "arc challenge",
      "raw_value": "34.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "arc-easy",
    "score": 0.581,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "arc easy",
      "raw_value": "58.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "boolq",
    "score": 0.607,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "boolq",
      "raw_value": "60.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "hellaswag",
    "score": 0.625,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "hellaswag",
      "raw_value": "62.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "openbookqa",
    "score": 0.464,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "openbookqa",
      "raw_value": "46.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "piqa",
    "score": 0.737,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "piqa",
      "raw_value": "73.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "sciq",
    "score": 0.881,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "sciq",
      "raw_value": "88.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-1b",
    "bench": "winogrande",
    "score": 0.589,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-1B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-1B",
      "column_label": "winogrande",
      "raw_value": "58.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "arc-challenge",
    "score": 0.475,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "arc challenge",
      "raw_value": "47.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "arc-easy",
    "score": 0.704,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "arc easy",
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "boolq",
    "score": 0.746,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "boolq",
      "raw_value": "74.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "hellaswag",
    "score": 0.759,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "hellaswag",
      "raw_value": "75.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "openbookqa",
    "score": 0.53,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "openbookqa",
      "raw_value": "53.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "piqa",
    "score": 0.785,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "piqa",
      "raw_value": "78.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "sciq",
    "score": 0.939,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "sciq",
      "raw_value": "93.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "falcon-7b",
    "bench": "winogrande",
    "score": 0.689,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Falcon-7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Falcon-7B",
      "column_label": "winogrande",
      "raw_value": "68.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-challenge",
    "score": 0.445,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "arc challenge",
      "raw_value": "44.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-easy",
    "score": 0.679,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "arc easy",
      "raw_value": "67.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "boolq",
    "score": 0.754,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "boolq",
      "raw_value": "75.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "hellaswag",
    "score": 0.762,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "hellaswag",
      "raw_value": "76.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "openbookqa",
    "score": 0.512,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "openbookqa",
      "raw_value": "51.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "piqa",
    "score": 0.772,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "piqa",
      "raw_value": "77.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "sciq",
    "score": 0.939,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "sciq",
      "raw_value": "93.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "winogrande",
    "score": 0.705,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'LLaMA 7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "winogrande",
      "raw_value": "70.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.485,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "arc challenge",
      "raw_value": "48.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-easy",
    "score": 0.695,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "arc easy",
      "raw_value": "69.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "boolq",
    "score": 0.802,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "boolq",
      "raw_value": "80.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "hellaswag",
    "score": 0.768,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "hellaswag",
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "openbookqa",
    "score": 0.484,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "openbookqa",
      "raw_value": "48.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "piqa",
    "score": 0.767,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "piqa",
      "raw_value": "76.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "sciq",
    "score": 0.945,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "sciq",
      "raw_value": "94.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "winogrande",
    "score": 0.694,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Llama 2 7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Base (pretrained) Llama 2 7B. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama 2 7B",
      "column_label": "winogrande",
      "raw_value": "69.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "arc-challenge",
    "score": 0.465,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "arc challenge",
      "raw_value": "46.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "arc-easy",
    "score": 0.705,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "arc easy",
      "raw_value": "70.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "boolq",
    "score": 0.742,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "boolq",
      "raw_value": "74.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "hellaswag",
    "score": 0.776,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "hellaswag",
      "raw_value": "77.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "openbookqa",
    "score": 0.486,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "openbookqa",
      "raw_value": "48.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "piqa",
    "score": 0.773,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "piqa",
      "raw_value": "77.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "sciq",
    "score": 0.937,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "sciq",
      "raw_value": "93.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "mpt-7b",
    "bench": "winogrande",
    "score": 0.699,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'MPT-7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "MPT-7B",
      "column_label": "winogrande",
      "raw_value": "69.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-challenge",
    "score": 0.441,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "arc challenge",
      "raw_value": "44.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-easy",
    "score": 0.619,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "arc easy",
      "raw_value": "61.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "boolq",
    "score": 0.611,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "boolq",
      "raw_value": "61.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "hellaswag",
    "score": 0.638,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "hellaswag",
      "raw_value": "63.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "openbookqa",
    "score": 0.45,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "openbookqa",
      "raw_value": "45.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "piqa",
    "score": 0.751,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "piqa",
      "raw_value": "75.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "sciq",
    "score": 0.911,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "sciq",
      "raw_value": "91.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "pythia-6-9b",
    "bench": "winogrande",
    "score": 0.62,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'Pythia 6.9B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Pythia 6.9B",
      "column_label": "winogrande",
      "raw_value": "62.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "arc-challenge",
    "score": 0.428,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "arc challenge",
      "raw_value": "42.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "arc-easy",
    "score": 0.684,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "arc easy",
      "raw_value": "68.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "boolq",
    "score": 0.686,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "boolq",
      "raw_value": "68.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "hellaswag",
    "score": 0.703,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "hellaswag",
      "raw_value": "70.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "openbookqa",
    "score": 0.494,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "openbookqa",
      "raw_value": "49.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "piqa",
    "score": 0.76,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "piqa",
      "raw_value": "76.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "sciq",
    "score": 0.929,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "sciq",
      "raw_value": "92.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "winogrande",
    "score": 0.647,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "independent",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'RPJ-INCITE-7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. Labelled 'RPJ-INCITE-7B' in the paper. ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "RPJ-INCITE-7B",
      "column_label": "winogrande",
      "raw_value": "64.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "arc-challenge",
    "score": 0.485,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'arc challenge'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "arc challenge",
      "raw_value": "48.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "arc-easy",
    "score": 0.654,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'arc easy'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "arc easy",
      "raw_value": "65.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "boolq",
    "score": 0.734,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'boolq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "boolq",
      "raw_value": "73.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "hellaswag",
    "score": 0.764,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'hellaswag'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "hellaswag",
      "raw_value": "76.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "openbookqa",
    "score": 0.504,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'openbookqa'. Zero-shot rank classification with AI2's Catwalk pipeline (unconditional normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "openbookqa",
      "raw_value": "50.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "piqa",
    "score": 0.784,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'piqa'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "piqa",
      "raw_value": "78.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "sciq",
    "score": 0.938,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'sciq'. Zero-shot rank classification with AI2's Catwalk pipeline (no normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "sciq",
      "raw_value": "93.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "olmo-7b",
    "bench": "winogrande",
    "score": 0.679,
    "source": "https://arxiv.org/html/2402.00838v4",
    "source_id": "paper-2402.00838v4",
    "by": "lab",
    "date": "2024-06-07",
    "mode": "base",
    "note": "OLMo paper (arXiv:2402.00838v4) Table 3, row 'OLMo-7B', column 'winogrande'. Zero-shot rank classification with AI2's Catwalk pipeline (per-token normalisation); baselines run by the OLMo authors with the same fixed pipeline. OLMo-7B results are for the 2.46T-token final checkpoint (allenai/OLMo-7B main). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "OLMo-7B",
      "column_label": "winogrande",
      "raw_value": "67.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "winogrande",
    "score": 0.691,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama2-7B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "WinoGrande",
      "raw_value": "0.691",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-easy",
    "score": 0.763,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama2-7B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "ARC-Easy",
      "raw_value": "0.763",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.434,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama2-7B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.434",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "boolq",
    "score": 0.779,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama2-7B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "BoolQ",
      "raw_value": "0.779",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "social-iqa",
    "score": 0.48,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama2-7B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "SIQA",
      "raw_value": "0.480",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "winogrande",
    "score": 0.669,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama-7B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "WinoGrande",
      "raw_value": "0.669",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-easy",
    "score": 0.682,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama-7B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "ARC-Easy",
      "raw_value": "0.682",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-challenge",
    "score": 0.385,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama-7B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.385",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "boolq",
    "score": 0.732,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama-7B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "BoolQ",
      "raw_value": "0.732",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "social-iqa",
    "score": 0.466,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Llama-7B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "SIQA",
      "raw_value": "0.466",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "winogrande",
    "score": 0.68,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'MPT-7B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "WinoGrande",
      "raw_value": "0.680",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "arc-easy",
    "score": 0.749,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'MPT-7B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "ARC-Easy",
      "raw_value": "0.749",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "arc-challenge",
    "score": 0.405,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'MPT-7B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.405",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "boolq",
    "score": 0.739,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'MPT-7B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "BoolQ",
      "raw_value": "0.739",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "social-iqa",
    "score": 0.451,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'MPT-7B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "SIQA",
      "raw_value": "0.451",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "winogrande",
    "score": 0.662,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-7B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "WinoGrande",
      "raw_value": "0.662",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "arc-easy",
    "score": 0.719,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-7B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "ARC-Easy",
      "raw_value": "0.719",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "arc-challenge",
    "score": 0.363,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-7B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.363",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "boolq",
    "score": 0.685,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-7B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "BoolQ",
      "raw_value": "0.685",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "social-iqa",
    "score": 0.452,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-7B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "SIQA",
      "raw_value": "0.452",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "winogrande",
    "score": 0.607,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-rw-1.3B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "WinoGrande",
      "raw_value": "0.607",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "arc-easy",
    "score": 0.633,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-rw-1.3B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "ARC-Easy",
      "raw_value": "0.633",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "arc-challenge",
    "score": 0.282,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-rw-1.3B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.282",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "boolq",
    "score": 0.632,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-rw-1.3B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "BoolQ",
      "raw_value": "0.632",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "social-iqa",
    "score": 0.405,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'Falcon-rw-1.3B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "SIQA",
      "raw_value": "0.405",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "winogrande",
    "score": 0.61,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'OPT-1.3B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "WinoGrande",
      "raw_value": "0.610",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "arc-easy",
    "score": 0.57,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'OPT-1.3B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "ARC-Easy",
      "raw_value": "0.570",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "arc-challenge",
    "score": 0.232,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'OPT-1.3B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.232",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "boolq",
    "score": 0.596,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'OPT-1.3B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "BoolQ",
      "raw_value": "0.596",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "winogrande",
    "score": 0.577,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT-Neo-2.7B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "WinoGrande",
      "raw_value": "0.577",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "arc-easy",
    "score": 0.611,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT-Neo-2.7B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "ARC-Easy",
      "raw_value": "0.611",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "arc-challenge",
    "score": 0.274,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT-Neo-2.7B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.274",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "boolq",
    "score": 0.618,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT-Neo-2.7B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "BoolQ",
      "raw_value": "0.618",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "social-iqa",
    "score": 0.4,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT-Neo-2.7B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "SIQA",
      "raw_value": "0.400",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "winogrande",
    "score": 0.583,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT2-XL-1.5B', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "WinoGrande",
      "raw_value": "0.583",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "arc-easy",
    "score": 0.583,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT2-XL-1.5B', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "ARC-Easy",
      "raw_value": "0.583",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "arc-challenge",
    "score": 0.25,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT2-XL-1.5B', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "ARC-Challenge",
      "raw_value": "0.250",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "boolq",
    "score": 0.618,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT2-XL-1.5B', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "BoolQ",
      "raw_value": "0.618",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "social-iqa",
    "score": 0.394,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'GPT2-XL-1.5B', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "SIQA",
      "raw_value": "0.394",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "winogrande",
    "score": 0.734,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'phi-1.5 (1.3B)', column 'WinoGrande'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "WinoGrande",
      "raw_value": "0.734",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "arc-easy",
    "score": 0.756,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'phi-1.5 (1.3B)', column 'ARC-Easy'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "ARC-Easy",
      "raw_value": "0.756",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "arc-challenge",
    "score": 0.444,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'phi-1.5 (1.3B)', column 'ARC-Challenge'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "ARC-Challenge",
      "raw_value": "0.444",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "boolq",
    "score": 0.758,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'phi-1.5 (1.3B)', column 'BoolQ'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "BoolQ",
      "raw_value": "0.758",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "social-iqa",
    "score": 0.526,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 2 (Common Sense Reasoning Benchmarks), row 'phi-1.5 (1.3B)', column 'SIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 2 (Common Sense Reasoning Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "SIQA",
      "raw_value": "0.526",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "piqa",
    "score": 0.781,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama2-7B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "PIQA",
      "raw_value": "0.781",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "hellaswag",
    "score": 0.571,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama2-7B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "Hellaswag",
      "raw_value": "0.571",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "openbookqa",
    "score": 0.314,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama2-7B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Llama2-7B'; values match the pretrained (base) checkpoint's harness profile and the table compares base models (the chat model Vicuna is listed separately). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama2-7B",
      "column_label": "OpenbookQA",
      "raw_value": "0.314",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "piqa",
    "score": 0.779,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama-7B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "PIQA",
      "raw_value": "0.779",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "hellaswag",
    "score": 0.562,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama-7B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "Hellaswag",
      "raw_value": "0.562",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "openbookqa",
    "score": 0.284,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Llama-7B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Llama-7B",
      "column_label": "OpenbookQA",
      "raw_value": "0.284",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "piqa",
    "score": 0.789,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'MPT-7B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "PIQA",
      "raw_value": "0.789",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "hellaswag",
    "score": 0.571,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'MPT-7B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "Hellaswag",
      "raw_value": "0.571",
      "scale_divisor": 1
    }
  },
  {
    "model": "mpt-7b",
    "bench": "openbookqa",
    "score": 0.314,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'MPT-7B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "MPT-7B",
      "column_label": "OpenbookQA",
      "raw_value": "0.314",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "piqa",
    "score": 0.794,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-7B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "PIQA",
      "raw_value": "0.794",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "hellaswag",
    "score": 0.542,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-7B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "Hellaswag",
      "raw_value": "0.542",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-7b",
    "bench": "openbookqa",
    "score": 0.32,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-7B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-7B",
      "column_label": "OpenbookQA",
      "raw_value": "0.320",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "piqa",
    "score": 0.747,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-rw-1.3B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "PIQA",
      "raw_value": "0.747",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "hellaswag",
    "score": 0.466,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-rw-1.3B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "Hellaswag",
      "raw_value": "0.466",
      "scale_divisor": 1
    }
  },
  {
    "model": "falcon-rw-1b",
    "bench": "openbookqa",
    "score": 0.244,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'Falcon-rw-1.3B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). Labelled 'Falcon-rw-1.3B' = tiiuae/falcon-rw-1b (1.3B parameters). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "Falcon-rw-1.3B",
      "column_label": "OpenbookQA",
      "raw_value": "0.244",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "piqa",
    "score": 0.69,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'OPT-1.3B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "PIQA",
      "raw_value": "0.690",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "hellaswag",
    "score": 0.415,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'OPT-1.3B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "Hellaswag",
      "raw_value": "0.415",
      "scale_divisor": 1
    }
  },
  {
    "model": "opt-1-3b",
    "bench": "openbookqa",
    "score": 0.24,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'OPT-1.3B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "OPT-1.3B",
      "column_label": "OpenbookQA",
      "raw_value": "0.240",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "piqa",
    "score": 0.729,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT-Neo-2.7B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "PIQA",
      "raw_value": "0.729",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "hellaswag",
    "score": 0.427,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT-Neo-2.7B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "Hellaswag",
      "raw_value": "0.427",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "openbookqa",
    "score": 0.232,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT-Neo-2.7B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT-Neo-2.7B",
      "column_label": "OpenbookQA",
      "raw_value": "0.232",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "piqa",
    "score": 0.705,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT2-XL-1.5B', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "PIQA",
      "raw_value": "0.705",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "hellaswag",
    "score": 0.4,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT2-XL-1.5B', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "Hellaswag",
      "raw_value": "0.400",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "openbookqa",
    "score": 0.224,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "independent",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'GPT2-XL-1.5B', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "GPT2-XL-1.5B",
      "column_label": "OpenbookQA",
      "raw_value": "0.224",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "piqa",
    "score": 0.766,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'phi-1.5 (1.3B)', column 'PIQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "PIQA",
      "raw_value": "0.766",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "hellaswag",
    "score": 0.476,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'phi-1.5 (1.3B)', column 'Hellaswag'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). HellaSwag plain accuracy (not acc_norm). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "Hellaswag",
      "raw_value": "0.476",
      "scale_divisor": 1
    }
  },
  {
    "model": "phi-1-5",
    "bench": "openbookqa",
    "score": 0.372,
    "source": "https://arxiv.org/html/2309.05463v1",
    "source_id": "paper-2309.05463v1",
    "by": "lab",
    "date": "2023-09-11",
    "mode": "base",
    "note": "phi-1.5 report (arXiv:2309.05463v1) Table 3 (Language Understanding and Knowledge Benchmarks), row 'phi-1.5 (1.3B)', column 'OpenbookQA'. Zero-shot accuracy with LM-Eval Harness; baselines re-run by the authors ('numbers are from our own evaluation pipeline'). ",
    "evidence": {
      "kind": "table",
      "table": "Table 3 (Language Understanding and Knowledge Benchmarks)",
      "row_label": "phi-1.5 (1.3B)",
      "column_label": "OpenbookQA",
      "raw_value": "0.372",
      "scale_divisor": 1
    }
  },
  {
    "model": "llama-7b",
    "bench": "boolq",
    "score": 0.765,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'BoolQ'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "BoolQ",
      "raw_value": "76.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "piqa",
    "score": 0.798,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'PIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "PIQA",
      "raw_value": "79.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "social-iqa",
    "score": 0.489,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'SIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "SIQA",
      "raw_value": "48.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "hellaswag",
    "score": 0.761,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'HellaSwag'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "HellaSwag",
      "raw_value": "76.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "winogrande",
    "score": 0.701,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'WinoGrande'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "WinoGrande",
      "raw_value": "70.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-easy",
    "score": 0.728,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'ARC-e'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "ARC-e",
      "raw_value": "72.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "arc-challenge",
    "score": 0.476,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'ARC-c'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "ARC-c",
      "raw_value": "47.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "openbookqa",
    "score": 0.572,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 7B', column 'OBQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 7B",
      "column_label": "OBQA",
      "raw_value": "57.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "boolq",
    "score": 0.781,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'BoolQ'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "BoolQ",
      "raw_value": "78.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "piqa",
    "score": 0.801,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'PIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "PIQA",
      "raw_value": "80.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "social-iqa",
    "score": 0.504,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'SIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "SIQA",
      "raw_value": "50.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "hellaswag",
    "score": 0.792,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'HellaSwag'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "HellaSwag",
      "raw_value": "79.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "winogrande",
    "score": 0.73,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'WinoGrande'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "WinoGrande",
      "raw_value": "73.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "arc-easy",
    "score": 0.748,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'ARC-e'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "ARC-e",
      "raw_value": "74.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "arc-challenge",
    "score": 0.527,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'ARC-c'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "ARC-c",
      "raw_value": "52.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "openbookqa",
    "score": 0.564,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 13B', column 'OBQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 13B",
      "column_label": "OBQA",
      "raw_value": "56.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "boolq",
    "score": 0.831,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'BoolQ'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "BoolQ",
      "raw_value": "83.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "piqa",
    "score": 0.823,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'PIQA'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "PIQA",
      "raw_value": "82.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "social-iqa",
    "score": 0.504,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'SIQA'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "SIQA",
      "raw_value": "50.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "hellaswag",
    "score": 0.828,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'HellaSwag'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "HellaSwag",
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "winogrande",
    "score": 0.76,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'WinoGrande'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "WinoGrande",
      "raw_value": "76.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "arc-easy",
    "score": 0.8,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'ARC-e'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "ARC-e",
      "raw_value": "80.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "arc-challenge",
    "score": 0.578,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'ARC-c'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "ARC-c",
      "raw_value": "57.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "openbookqa",
    "score": 0.586,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 33B', column 'OBQA'. Meta's own zero-shot evaluation of its model. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 33B",
      "column_label": "OBQA",
      "raw_value": "58.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "boolq",
    "score": 0.853,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'BoolQ'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "BoolQ",
      "raw_value": "85.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "piqa",
    "score": 0.828,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'PIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "PIQA",
      "raw_value": "82.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "social-iqa",
    "score": 0.523,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'SIQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "SIQA",
      "raw_value": "52.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "hellaswag",
    "score": 0.842,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'HellaSwag'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "HellaSwag",
      "raw_value": "84.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "winogrande",
    "score": 0.77,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'WinoGrande'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "WinoGrande",
      "raw_value": "77.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "arc-easy",
    "score": 0.789,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'ARC-e'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "ARC-e",
      "raw_value": "78.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "arc-challenge",
    "score": 0.56,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'ARC-c'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "ARC-c",
      "raw_value": "56.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "openbookqa",
    "score": 0.602,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper (arXiv:2302.13971v1) Table 3 'Zero-shot performance on Common Sense Reasoning tasks', row 'LLaMA 65B', column 'OBQA'. Meta's own zero-shot evaluation of its model.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "LLaMA 65B",
      "column_label": "OBQA",
      "raw_value": "60.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "math",
    "score": 0.029,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 7B', column 'MATH' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 7B",
      "column_label": "MATH",
      "raw_value": "2.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "gsm8k",
    "score": 0.11,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 7B', column 'GSM8k' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 7B",
      "column_label": "GSM8k",
      "raw_value": "11.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "math",
    "score": 0.039,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 13B', column 'MATH' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 13B",
      "column_label": "MATH",
      "raw_value": "3.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "gsm8k",
    "score": 0.178,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 13B', column 'GSM8k' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 13B",
      "column_label": "GSM8k",
      "raw_value": "17.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "math",
    "score": 0.071,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 33B', column 'MATH' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 33B",
      "column_label": "MATH",
      "raw_value": "7.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "gsm8k",
    "score": 0.356,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 33B', column 'GSM8k' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 33B",
      "column_label": "GSM8k",
      "raw_value": "35.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "math",
    "score": 0.106,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 65B', column 'MATH' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 65B",
      "column_label": "MATH",
      "raw_value": "10.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "gsm8k",
    "score": 0.509,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 7 (quantitative reasoning), row 'LLaMA 65B', column 'GSM8k' (single-sample score, not the '+maj1@k' column). Minerva-style prompting as described in the paper; MATH is the full MATH test set.",
    "evidence": {
      "kind": "table",
      "table": "Table 7",
      "row_label": "LLaMA 65B",
      "column_label": "GSM8k",
      "raw_value": "50.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "humaneval",
    "score": 0.105,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 7B', column 'HumanEval pass@1' (zero-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 7B",
      "column_label": "HumanEval pass@1",
      "raw_value": "10.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "mbpp",
    "score": 0.177,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 7B', column 'MBPP pass@1' (3-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 7B",
      "column_label": "MBPP pass@1",
      "raw_value": "17.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "humaneval",
    "score": 0.158,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 13B', column 'HumanEval pass@1' (zero-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 13B",
      "column_label": "HumanEval pass@1",
      "raw_value": "15.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "mbpp",
    "score": 0.22,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 13B', column 'MBPP pass@1' (3-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 13B",
      "column_label": "MBPP pass@1",
      "raw_value": "22.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "humaneval",
    "score": 0.217,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 33B', column 'HumanEval pass@1' (zero-shot). Original HumanEval/MBPP, not the + versions. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 33B",
      "column_label": "HumanEval pass@1",
      "raw_value": "21.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "mbpp",
    "score": 0.302,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 33B', column 'MBPP pass@1' (3-shot). Original HumanEval/MBPP, not the + versions. Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 33B",
      "column_label": "MBPP pass@1",
      "raw_value": "30.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "humaneval",
    "score": 0.237,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 65B', column 'HumanEval pass@1' (zero-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 65B",
      "column_label": "HumanEval pass@1",
      "raw_value": "23.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "mbpp",
    "score": 0.377,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 8 (code generation), row 'LLaMA 65B', column 'MBPP pass@1' (3-shot). Original HumanEval/MBPP, not the + versions.",
    "evidence": {
      "kind": "table",
      "table": "Table 8",
      "row_label": "LLaMA 65B",
      "column_label": "MBPP pass@1",
      "raw_value": "37.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-7b",
    "bench": "mmlu",
    "score": 0.351,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 9 (MMLU, five-shot accuracy), row 'LLaMA 7B', column 'Average' (57 subjects).",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "LLaMA 7B",
      "column_label": "Average",
      "raw_value": "35.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-13b",
    "bench": "mmlu",
    "score": 0.469,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 9 (MMLU, five-shot accuracy), row 'LLaMA 13B', column 'Average' (57 subjects).",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "LLaMA 13B",
      "column_label": "Average",
      "raw_value": "46.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-33b",
    "bench": "mmlu",
    "score": 0.578,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 9 (MMLU, five-shot accuracy), row 'LLaMA 33B', column 'Average' (57 subjects). Released as 'llama-30b'.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "LLaMA 33B",
      "column_label": "Average",
      "raw_value": "57.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-65b",
    "bench": "mmlu",
    "score": 0.634,
    "source": "https://arxiv.org/html/2302.13971v1",
    "source_id": "paper-2302.13971v1",
    "by": "lab",
    "date": "2023-02-27",
    "mode": "base",
    "note": "LLaMA paper Table 9 (MMLU, five-shot accuracy), row 'LLaMA 65B', column 'Average' (57 subjects).",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "LLaMA 65B",
      "column_label": "Average",
      "raw_value": "63.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom-560m",
    "bench": "humaneval",
    "score": 0.0082,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM-560M', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM-560M",
      "column_label": "pass@k k=1",
      "raw_value": "0.82%",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom-1b1",
    "bench": "humaneval",
    "score": 0.0248,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM-1.1B', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM-1.1B",
      "column_label": "pass@k k=1",
      "raw_value": "2.48%",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom-1b7",
    "bench": "humaneval",
    "score": 0.0403,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM-1.7B', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM-1.7B",
      "column_label": "pass@k k=1",
      "raw_value": "4.03%",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom-3b",
    "bench": "humaneval",
    "score": 0.0648,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM-3B', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM-3B",
      "column_label": "pass@k k=1",
      "raw_value": "6.48%",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom-7b1",
    "bench": "humaneval",
    "score": 0.0773,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM-7.1B', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM-7.1B",
      "column_label": "pass@k k=1",
      "raw_value": "7.73%",
      "scale_divisor": 100
    }
  },
  {
    "model": "bloom",
    "bench": "humaneval",
    "score": 0.1552,
    "source": "https://arxiv.org/html/2211.05100v4",
    "source_id": "paper-2211.05100v4",
    "by": "lab",
    "date": "2023-06-27",
    "mode": "base",
    "note": "BLOOM paper (arXiv:2211.05100v4) Table 9 'Performance on HumanEval', row 'BLOOM', column 'k=1' (pass@1). BigScience's own evaluation of its pretrained (not BLOOMZ) model.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "BLOOM",
      "column_label": "pass@k k=1",
      "raw_value": "15.52%",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-small",
    "bench": "mmlu",
    "score": 0.287,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Small', column 'MMLU Direct' (five-shot, direct answer; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Small",
      "column_label": "MMLU Direct",
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-small",
    "bench": "bbh",
    "score": 0.291,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Small', column 'BBH Direct' (three-shot, direct answer, 23 tasks; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Small",
      "column_label": "BBH Direct",
      "raw_value": "29.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-base",
    "bench": "mmlu",
    "score": 0.359,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Base', column 'MMLU Direct' (five-shot, direct answer; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Base",
      "column_label": "MMLU Direct",
      "raw_value": "35.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-base",
    "bench": "bbh",
    "score": 0.313,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Base', column 'BBH Direct' (three-shot, direct answer, 23 tasks; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Base",
      "column_label": "BBH Direct",
      "raw_value": "31.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-large",
    "bench": "mmlu",
    "score": 0.451,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Large', column 'MMLU Direct' (five-shot, direct answer; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Large",
      "column_label": "MMLU Direct",
      "raw_value": "45.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-large",
    "bench": "bbh",
    "score": 0.375,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Large', column 'BBH Direct' (three-shot, direct answer, 23 tasks; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-Large",
      "column_label": "BBH Direct",
      "raw_value": "37.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-xl",
    "bench": "mmlu",
    "score": 0.524,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XL', column 'MMLU Direct' (five-shot, direct answer; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-XL",
      "column_label": "MMLU Direct",
      "raw_value": "52.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-xl",
    "bench": "bbh",
    "score": 0.41,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XL', column 'BBH Direct' (three-shot, direct answer, 23 tasks; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-XL",
      "column_label": "BBH Direct",
      "raw_value": "41.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-xxl",
    "bench": "mmlu",
    "score": 0.551,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XXL', column 'MMLU Direct' (five-shot, direct answer; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-XXL",
      "column_label": "MMLU Direct",
      "raw_value": "55.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "flan-t5-xxl",
    "bench": "bbh",
    "score": 0.453,
    "source": "https://arxiv.org/html/2210.11416v5",
    "source_id": "paper-2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "mode": "instruct",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XXL', column 'BBH Direct' (three-shot, direct answer, 23 tasks; exact-match accuracy). Google's own evaluation of its released instruction-finetuned checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Flan-T5-XXL",
      "column_label": "BBH Direct",
      "raw_value": "45.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-1",
    "bench": "humaneval",
    "score": 0.506,
    "source": "https://arxiv.org/html/2306.11644v2",
    "source_id": "paper-2306.11644v2",
    "by": "lab",
    "date": "2023-10-02",
    "mode": "base",
    "note": "phi-1 paper (arXiv:2306.11644v2) Table 1, row '2023 Jun | phi-1', column 'HumanEval (Pass@1)'. Microsoft's own evaluation of phi-1 (1.3B); original HumanEval/MBPP.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "phi-1",
      "column_label": "HumanEval (Pass@1)",
      "raw_value": "50.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-1",
    "bench": "mbpp",
    "score": 0.555,
    "source": "https://arxiv.org/html/2306.11644v2",
    "source_id": "paper-2306.11644v2",
    "by": "lab",
    "date": "2023-10-02",
    "mode": "base",
    "note": "phi-1 paper (arXiv:2306.11644v2) Table 1, row '2023 Jun | phi-1', column 'MBPP (Pass@1)'. Microsoft's own evaluation of phi-1 (1.3B); original HumanEval/MBPP.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "phi-1",
      "column_label": "MBPP (Pass@1)",
      "raw_value": "55.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "text-davinci-001",
    "bench": "eq-bench-v1",
    "score": 0.1519,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison', row 'OpenAI text-davinci-001', 'EQ-Bench Score' 15.19 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author via the OpenAI API in late 2023. Same value appears in Tables 3-5.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI text-davinci-001",
      "column_label": "EQ-Bench Score",
      "raw_value": "15.19",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "mmlu",
    "score": 0.444,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'MMLU' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "MMLU",
      "raw_value": "44.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "hellaswag",
    "score": 0.771,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'HellaSwag' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "HellaSwag",
      "raw_value": "77.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "winogrande",
    "score": 0.695,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'WinoG' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "WinoG",
      "raw_value": "69.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "piqa",
    "score": 0.779,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'PIQA' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "PIQA",
      "raw_value": "77.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-easy",
    "score": 0.687,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'Arc-e' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "Arc-e",
      "raw_value": "68.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.432,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'Arc-c' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "Arc-c",
      "raw_value": "43.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "triviaqa",
    "score": 0.638,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'TriviaQA' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "TriviaQA",
      "raw_value": "63.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "humaneval",
    "score": 0.116,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'HumanEval' (0-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "HumanEval",
      "raw_value": "11.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "mbpp",
    "score": 0.261,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'MBPP' (3-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "MBPP",
      "raw_value": "26.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "math",
    "score": 0.039,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'MATH' (4-shot, maj@4, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "MATH",
      "raw_value": "3.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b",
    "bench": "gsm8k",
    "score": 0.16,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'GSM8K' (8-shot, maj@8, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 7B",
      "column_label": "GSM8K",
      "raw_value": "16.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "mmlu",
    "score": 0.556,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'MMLU' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "MMLU",
      "raw_value": "55.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "hellaswag",
    "score": 0.807,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'HellaSwag' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "HellaSwag",
      "raw_value": "80.7%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "winogrande",
    "score": 0.729,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'WinoG' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "WinoG",
      "raw_value": "72.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "piqa",
    "score": 0.808,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'PIQA' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "PIQA",
      "raw_value": "80.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-easy",
    "score": 0.752,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'Arc-e' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "Arc-e",
      "raw_value": "75.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-challenge",
    "score": 0.488,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'Arc-c' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "Arc-c",
      "raw_value": "48.8%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "triviaqa",
    "score": 0.696,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'TriviaQA' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "TriviaQA",
      "raw_value": "69.6%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "humaneval",
    "score": 0.189,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'HumanEval' (0-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "HumanEval",
      "raw_value": "18.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "mbpp",
    "score": 0.354,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'MBPP' (3-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "MBPP",
      "raw_value": "35.4%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "math",
    "score": 0.06,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'MATH' (4-shot, maj@4, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "MATH",
      "raw_value": "6.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b",
    "bench": "gsm8k",
    "score": 0.343,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'GSM8K' (8-shot, maj@8, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Base (pretrained) Llama 2, not Llama 2 Chat.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "LLaMA 2 13B",
      "column_label": "GSM8K",
      "raw_value": "34.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mmlu",
    "score": 0.601,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'MMLU' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "MMLU",
      "raw_value": "60.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "hellaswag",
    "score": 0.813,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'HellaSwag' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "HellaSwag",
      "raw_value": "81.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "winogrande",
    "score": 0.753,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'WinoG' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "WinoG",
      "raw_value": "75.3%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "piqa",
    "score": 0.83,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'PIQA' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "PIQA",
      "raw_value": "83.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-easy",
    "score": 0.8,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'Arc-e' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "Arc-e",
      "raw_value": "80.0%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-challenge",
    "score": 0.555,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'Arc-c' (0-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "Arc-c",
      "raw_value": "55.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "triviaqa",
    "score": 0.699,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'TriviaQA' (5-shot, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "TriviaQA",
      "raw_value": "69.9%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "humaneval",
    "score": 0.305,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'HumanEval' (0-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "HumanEval",
      "raw_value": "30.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mbpp",
    "score": 0.475,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'MBPP' (3-shot, pass@1, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "MBPP",
      "raw_value": "47.5%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "math",
    "score": 0.131,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'MATH' (4-shot, maj@4, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "MATH",
      "raw_value": "13.1%",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "gsm8k",
    "score": 0.522,
    "source": "https://arxiv.org/html/2310.06825v1",
    "source_id": "paper-2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "mode": "base",
    "note": "Mistral 7B paper (arXiv:2310.06825v1) Table 2, row 'Mistral 7B' (Pretrained), column 'GSM8K' (8-shot, maj@8, as stated in Section 3). All models re-evaluated by Mistral AI with its own pipeline. Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral 7B",
      "column_label": "GSM8K",
      "raw_value": "52.2%",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "copa",
    "score": 0.61,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'COPA' (COPA), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "COPA",
      "column_label": "125M LM",
      "raw_value": "0.610",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "copa",
    "score": 0.67,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'COPA' (COPA), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "COPA",
      "column_label": "350M LM",
      "raw_value": "0.670",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "copa",
    "score": 0.698,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'COPA' (COPA), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "COPA",
      "column_label": "760M LM",
      "raw_value": "0.698",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "copa",
    "score": 0.69,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'COPA' (COPA), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "COPA",
      "column_label": "1.6B LM",
      "raw_value": "0.690",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "storycloze",
    "score": 0.6,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'SC' (StoryCloze), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "SC",
      "column_label": "125M LM",
      "raw_value": "0.600",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "storycloze",
    "score": 0.63,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'SC' (StoryCloze), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "SC",
      "column_label": "350M LM",
      "raw_value": "0.630",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "storycloze",
    "score": 0.661,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'SC' (StoryCloze), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "SC",
      "column_label": "760M LM",
      "raw_value": "0.661",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "storycloze",
    "score": 0.676,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'SC' (StoryCloze), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "SC",
      "column_label": "1.6B LM",
      "raw_value": "0.676",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "hellaswag",
    "score": 0.286,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'HS' (HellaSwag), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "HS",
      "column_label": "125M LM",
      "raw_value": "0.286",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "hellaswag",
    "score": 0.322,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'HS' (HellaSwag), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "HS",
      "column_label": "350M LM",
      "raw_value": "0.322",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "hellaswag",
    "score": 0.35,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'HS' (HellaSwag), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "HS",
      "column_label": "760M LM",
      "raw_value": "0.350",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "hellaswag",
    "score": 0.384,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'HS' (HellaSwag), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "HS",
      "column_label": "1.6B LM",
      "raw_value": "0.384",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "arc-easy",
    "score": 0.429,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-E' (ARC-Easy), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-E",
      "column_label": "125M LM",
      "raw_value": "0.429",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "arc-easy",
    "score": 0.494,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-E' (ARC-Easy), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-E",
      "column_label": "350M LM",
      "raw_value": "0.494",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "arc-easy",
    "score": 0.527,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-E' (ARC-Easy), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-E",
      "column_label": "760M LM",
      "raw_value": "0.527",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "arc-easy",
    "score": 0.562,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-E' (ARC-Easy), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-E",
      "column_label": "1.6B LM",
      "raw_value": "0.562",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "arc-challenge",
    "score": 0.201,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-C' (ARC-Challenge), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-C",
      "column_label": "125M LM",
      "raw_value": "0.201",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "arc-challenge",
    "score": 0.228,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-C' (ARC-Challenge), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-C",
      "column_label": "350M LM",
      "raw_value": "0.228",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "arc-challenge",
    "score": 0.231,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-C' (ARC-Challenge), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-C",
      "column_label": "760M LM",
      "raw_value": "0.231",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "arc-challenge",
    "score": 0.252,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'ARC-C' (ARC-Challenge), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "ARC-C",
      "column_label": "1.6B LM",
      "raw_value": "0.252",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "openbookqa",
    "score": 0.164,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'OBQA' (OpenBookQA), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "OBQA",
      "column_label": "125M LM",
      "raw_value": "0.164",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "openbookqa",
    "score": 0.186,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'OBQA' (OpenBookQA), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "OBQA",
      "column_label": "350M LM",
      "raw_value": "0.186",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "openbookqa",
    "score": 0.194,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'OBQA' (OpenBookQA), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "OBQA",
      "column_label": "760M LM",
      "raw_value": "0.194",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "openbookqa",
    "score": 0.224,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'OBQA' (OpenBookQA), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "OBQA",
      "column_label": "1.6B LM",
      "raw_value": "0.224",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-small",
    "bench": "commonsenseqa",
    "score": 0.255,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'CQA' (CommonsenseQA (dev)), column '125M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 125M = Hugging Face 'gpt2' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "CQA",
      "column_label": "125M LM",
      "raw_value": "0.255",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-medium",
    "bench": "commonsenseqa",
    "score": 0.309,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'CQA' (CommonsenseQA (dev)), column '350M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 350M = Hugging Face 'gpt2-medium' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "CQA",
      "column_label": "350M LM",
      "raw_value": "0.309",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-large",
    "bench": "commonsenseqa",
    "score": 0.333,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'CQA' (CommonsenseQA (dev)), column '760M' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 760M = Hugging Face 'gpt2-large' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "CQA",
      "column_label": "760M LM",
      "raw_value": "0.333",
      "scale_divisor": 1
    }
  },
  {
    "model": "gpt-2-xl",
    "bench": "commonsenseqa",
    "score": 0.386,
    "source": "https://aclanthology.org/2021.emnlp-main.564.pdf",
    "source_id": "acl-2021-emnlp-main-564",
    "by": "independent",
    "date": "2021-11-01",
    "mode": "base",
    "note": "Holtzman et al. (EMNLP 2021) Appendix A Table 6 'Multiple Choice Accuracy on GPT-2', row 'CQA' (CommonsenseQA (dev)), column '1.6B' / 'LM' (zero-shot, answer chosen by unnormalised conditional log-probability; templates in their Table 7). GPT-2 1.6B = Hugging Face 'gpt2-xl' checkpoint. Date: EMNLP 2021 (November 2021).",
    "evidence": {
      "kind": "table",
      "table": "Table 6 (Appendix A)",
      "row_label": "CQA",
      "column_label": "1.6B LM",
      "raw_value": "0.386",
      "scale_divisor": 1
    }
  },
  {
    "model": "parler-tts-large-v1",
    "bench": "ttsds-v1-default",
    "score": 0.8789123153,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key parler_tts_large.tar.gz; submission 'Parler TTS Large'; inference details: Used 100 random prompts from https://huggingface.co/datasets/parler-tts/libritts-r-filtered-speaker-descriptions. Identity: Submission 'Parler TTS Large' links parler-tts/parler-tts-large-v1; 100 random speaker descriptions from parler-tts/libritts-r-filtered-speaker-descriptions (description-prompted, not voice cloning).",
    "raw_value": 87.89123152842632,
    "scale_divisor": 100,
    "components": {
      "1": 95.5149921384996,
      "2": 86.42515281534912,
      "4": 75.09563213464236,
      "6": 94.5291490252142
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/parler_tts_large.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/parler_tts_large.json",
      "https://huggingface.co/parler-tts/parler-tts-large-v1"
    ],
    "source_model_label": "Parler TTS Large"
  },
  {
    "model": "parler-tts-mini-v0-1",
    "bench": "ttsds-v1-default",
    "score": 0.8562603911,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key parler_tts.tar.gz; submission 'Parler TTS Mini'; inference details: Used on description per speaker from https://huggingface.co/datasets/parler-tts/libritts_r_tags_tagged_10k_generated. Identity: Submission 'Parler TTS Mini' (org link only); rows present in the first results.csv commit 5c6d94d (2024-07-16), before Parler-TTS Mini v1 was published (2024-08-08), and prompts drawn from parler-tts/libritts_r_tags_tagged_10k_generated, the Mini v0.1 training annotations -> parler-tts/parler_tts_mini_v0.1.",
    "raw_value": 85.62603911098111,
    "scale_divisor": 100,
    "components": {
      "1": 94.6791462810385,
      "2": 81.04950591556164,
      "4": 74.09160210719313,
      "6": 92.68390214013118
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/parler_tts.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/parler_tts.json",
      "https://huggingface.co/datasets/ttsds/results/blob/5c6d94dd15ad4d9ba8f28c936eb40221349ff8e7/results.csv",
      "https://huggingface.co/parler-tts/parler_tts_mini_v0.1"
    ],
    "source_model_label": "Parler TTS Mini"
  },
  {
    "model": "pheme-300m",
    "bench": "ttsds-v1-default",
    "score": 0.8491123481,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key pheme.tar.gz; submission 'Pheme'; inference details: not supplied. Identity: Submission 'Pheme' links PolyAI/pheme, the 300M variant (the 100M variant is the separate PolyAI/pheme_small repo).",
    "raw_value": 84.91123481051297,
    "scale_divisor": 100,
    "components": {
      "1": 94.04385816439732,
      "2": 85.7486672603291,
      "4": 66.11636651399132,
      "6": 93.73604730333412
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/pheme.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/pheme.json",
      "https://huggingface.co/PolyAI/pheme"
    ],
    "source_model_label": "Pheme"
  },
  {
    "model": "styletts-2-libritts",
    "bench": "ttsds-v1-default",
    "score": 0.8803283897,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key styletts_2.tar.gz; submission 'StyleTTS 2'; inference details: not supplied. Identity: Submission 'StyleTTS 2' tagged Reference-based (Speaker) and conditioned on LibriTTS speaker references; the only released multi-speaker StyleTTS 2 checkpoint is yl4579/StyleTTS2-LibriTTS (the LJSpeech checkpoint is single-speaker).",
    "raw_value": 88.03283897304647,
    "scale_divisor": 100,
    "components": {
      "1": 93.71799967089886,
      "2": 89.62123516865168,
      "4": 71.54662604751235,
      "6": 97.24549500512298
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/styletts_2.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/styletts_2.json",
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ],
    "source_model_label": "StyleTTS 2"
  },
  {
    "model": "fish-speech-1-2",
    "bench": "ttsds-v1-default",
    "score": 0.8549456535,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key fish_speech.tar.gz; submission 'Fish Speech'; inference details: not supplied. Identity: Submission 'Fish Speech' links fishaudio/fish-speech-1.2; added 2024-08-09.",
    "raw_value": 85.49456534575503,
    "scale_divisor": 100,
    "components": {
      "1": 93.3567881245795,
      "2": 79.69032102572942,
      "4": 72.86706702610758,
      "6": 96.06408520660364
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/fish_speech.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/fish_speech.json",
      "https://huggingface.co/fishaudio/fish-speech-1.2"
    ],
    "source_model_label": "Fish Speech"
  },
  {
    "model": "amphion-valle-librilight-6k",
    "bench": "ttsds-v1-default",
    "score": 0.8368159302,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key amphion_vallev1.tar.gz; submission 'Amphion VALLEv1'; inference details: not supplied. Identity: Submission 'Amphion VALLEv1' links amphion/valle_librilight_6k (Amphion VALL-E v1 recipe reproduction, not Microsoft's unreleased VALL-E).",
    "raw_value": 83.68159302290015,
    "scale_divisor": 100,
    "components": {
      "1": 91.74638897515827,
      "2": 85.63738496022206,
      "4": 70.57304684415345,
      "6": 86.76955131206681
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_vallev1.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_vallev1.json",
      "https://huggingface.co/amphion/valle_librilight_6k"
    ],
    "source_model_label": "Amphion VALLEv1"
  },
  {
    "model": "openvoice-v1",
    "bench": "ttsds-v1-default",
    "score": 0.871418554,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key openvoice.tar.gz; submission 'OpenVoice'; inference details: not supplied. Identity: Submission 'OpenVoice' links myshell-ai/OpenVoice (V1; V2 is a separate submission).",
    "raw_value": 87.1418553994794,
    "scale_divisor": 100,
    "components": {
      "1": 91.72473157112938,
      "2": 92.31387467798204,
      "4": 68.80173497095757,
      "6": 95.72708037784862
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/openvoice.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/openvoice.json",
      "https://huggingface.co/myshell-ai/OpenVoice"
    ],
    "source_model_label": "OpenVoice"
  },
  {
    "model": "amphion-naturalspeech2",
    "bench": "ttsds-v1-default",
    "score": 0.820548397,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key amphion_naturalspeech2.tar.gz; submission 'Amphion NaturalSpeech2'; inference details: not supplied. Identity: Submission 'Amphion NaturalSpeech2' links the Amphion NaturalSpeech2 recipe/Space (Amphion reproduction; its released checkpoint is amphion/naturalspeech2_libritts). Microsoft's original NaturalSpeech 2 was never released.",
    "raw_value": 82.05483969624345,
    "scale_divisor": 100,
    "components": {
      "1": 88.87345384259388,
      "2": 77.73807529486058,
      "4": 67.81390895364515,
      "6": 93.79392069387418
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_naturalspeech2.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_naturalspeech2.json",
      "https://huggingface.co/amphion/naturalspeech2_libritts"
    ],
    "source_model_label": "Amphion NaturalSpeech2"
  },
  {
    "model": "vokan",
    "bench": "ttsds-v1-default",
    "score": 0.8454000317,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key vokan.tar.gz; submission 'Vokan'; inference details: not supplied. Identity: Submission 'Vokan' links ShoukanLabs/Vokan (StyleTTS 2 fine-tune).",
    "raw_value": 84.54000316946716,
    "scale_divisor": 100,
    "components": {
      "1": 88.57064692980998,
      "2": 84.9666840568015,
      "4": 69.07226520599738,
      "6": 95.55041648525973
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/vokan.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/vokan.json",
      "https://huggingface.co/ShoukanLabs/Vokan"
    ],
    "source_model_label": "Vokan"
  },
  {
    "model": "bark",
    "bench": "ttsds-v1-default",
    "score": 0.7308121226,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key bark.tar.gz; submission 'Bark'; inference details: Used https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer for prompts.. Identity: Submission 'Bark' links the suno/bark Space and suno-ai/bark code; prompts made with gitmylo/bark-voice-cloning-HuBERT-quantizer. Bark's default (full-size) text/coarse/fine models assumed: no small-model setting is mentioned.",
    "raw_value": 73.08121226370102,
    "scale_divisor": 100,
    "components": {
      "1": 81.28794368603677,
      "2": 71.53229754331544,
      "4": 69.10217994409112,
      "6": 70.40242788136078
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/bark.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/bark.json",
      "https://huggingface.co/suno/bark"
    ],
    "source_model_label": "Bark"
  },
  {
    "model": "amphion-valle-v2",
    "bench": "ttsds-v1-default",
    "score": 0.7031174218,
    "source": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv",
    "source_id": "speech-ttsds-v1-mit",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Derived official default TTSDS v1 composite, not an average of all 12 metrics. Source dataset key amphion_vallev2.tar.gz; submission 'Amphion VALLEv2'; inference details: Temperature 1, removed results with 0 length.. Identity: Submission 'Amphion VALLEv2' links amphion/valle (Amphion VALL-E v2 recipe reproduction); inference: temperature 1, zero-length outputs removed.",
    "raw_value": 70.31174217602151,
    "scale_divisor": 100,
    "components": {
      "1": 75.84749306072139,
      "2": 70.94343783420999,
      "4": 59.296106541280665,
      "6": 75.15993126787399
    },
    "identity_evidence": "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_vallev2.json",
    "identity_sources": [
      "https://huggingface.co/datasets/ttsds/requests/blob/9ca59ff46a60bae35f6d0ca42836546da603f70d/amphion_vallev2.json",
      "https://huggingface.co/amphion/valle"
    ],
    "source_model_label": "Amphion VALLEv2"
  },
  {
    "model": "bark",
    "bench": "ttsds2-listening-mos",
    "score": 0.372221,
    "raw_value": 2.488882,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'bark', computed from 279 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 1.783, Kids 2.494, Noisy 2.791, Wild 2.887. Identity (ttsds/systems 'bark'): suno/bark full-size models (text_2/coarse_2/fine_2; bark_small is a separate, unrated system) with GitMylo HuBERT-quantizer voice-cloning prompts. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 279,
    "per_domain": {
      "Clean": 1.783333,
      "Kids": 2.493827,
      "Noisy": 2.791045,
      "Wild": 2.887324
    },
    "source_model_label": "bark",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "bark",
    "bench": "ttsds2-listening-smos",
    "score": 0.377238,
    "raw_value": 2.508954,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'bark', computed from 193 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 2.608, Kids 1.878, Noisy 2.452, Wild 3.098. Identity (ttsds/systems 'bark'): suno/bark full-size models (text_2/coarse_2/fine_2; bark_small is a separate, unrated system) with GitMylo HuBERT-quantizer voice-cloning prompts. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 193,
    "per_domain": {
      "Clean": 2.607843,
      "Kids": 1.877551,
      "Noisy": 2.452381,
      "Wild": 3.098039
    },
    "source_model_label": "bark",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "ttsds2-listening-mos",
    "score": 0.603697,
    "raw_value": 3.41479,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'e2', computed from 267 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.441, Kids 3.815, Noisy 3.000, Wild 3.403. Identity (ttsds/systems 'e2'): SWivid/E2-TTS E2TTS_Base/model_1200000.safetensors (F5-TTS authors' E2 TTS reproduction; Microsoft's original is unreleased). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 267,
    "per_domain": {
      "Clean": 3.441176,
      "Kids": 3.815385,
      "Noisy": 3,
      "Wild": 3.402597
    },
    "source_model_label": "e2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "ttsds2-listening-smos",
    "score": 0.84128,
    "raw_value": 4.36512,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'e2', computed from 159 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.452, Kids 4.188, Noisy 4.286, Wild 4.535. Identity (ttsds/systems 'e2'): SWivid/E2-TTS E2TTS_Base/model_1200000.safetensors (F5-TTS authors' E2 TTS reproduction; Microsoft's original is unreleased). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 159,
    "per_domain": {
      "Clean": 4.452381,
      "Kids": 4.1875,
      "Noisy": 4.285714,
      "Wild": 4.534884
    },
    "source_model_label": "e2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "ttsds2-listening-mos",
    "score": 0.583459,
    "raw_value": 3.333838,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'f5', computed from 244 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.246, Kids 3.492, Noisy 3.080, Wild 3.517. Identity (ttsds/systems 'f5'): SWivid/F5-TTS F5TTS_Base/model_1200000.safetensors (F5-TTS v0 Base; not F5TTS_v1_Base, released 2025-03 after the 2025-01-01 cut-off). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 244,
    "per_domain": {
      "Clean": 3.246377,
      "Kids": 3.492308,
      "Noisy": 3.08,
      "Wild": 3.516667
    },
    "source_model_label": "f5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "ttsds2-listening-smos",
    "score": 0.775151,
    "raw_value": 4.100604,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'f5', computed from 150 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.360, Kids 3.600, Noisy 4.024, Wild 4.419. Identity (ttsds/systems 'f5'): SWivid/F5-TTS F5TTS_Base/model_1200000.safetensors (F5-TTS v0 Base; not F5TTS_v1_Base, released 2025-03 after the 2025-01-01 cut-off). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 150,
    "per_domain": {
      "Clean": 4.36,
      "Kids": 3.6,
      "Noisy": 4.02381,
      "Wild": 4.418605
    },
    "source_model_label": "f5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "fish-speech-1-5",
    "bench": "ttsds2-listening-mos",
    "score": 0.561014,
    "raw_value": 3.244056,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'fishspeech_1_5', computed from 218 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.034, Kids 3.400, Noisy 3.366, Wild 3.176. Identity (ttsds/systems 'fishspeech_1_5'): fishaudio/fish-speech-1.5 (firefly-gan-vq-fsq-8x1024-21hz decoder). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 218,
    "per_domain": {
      "Clean": 3.033898,
      "Kids": 3.4,
      "Noisy": 3.365854,
      "Wild": 3.176471
    },
    "source_model_label": "fishspeech_1_5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "fish-speech-1-5",
    "bench": "ttsds2-listening-smos",
    "score": 0.645802,
    "raw_value": 3.583206,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'fishspeech_1_5', computed from 153 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.535, Kids 3.463, Noisy 3.846, Wild 3.488. Identity (ttsds/systems 'fishspeech_1_5'): fishaudio/fish-speech-1.5 (firefly-gan-vq-fsq-8x1024-21hz decoder). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 153,
    "per_domain": {
      "Clean": 3.534884,
      "Kids": 3.463415,
      "Noisy": 3.846154,
      "Wild": 3.488372
    },
    "source_model_label": "fishspeech_1_5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "gpt-sovits-v2",
    "bench": "ttsds2-listening-mos",
    "score": 0.452629,
    "raw_value": 2.810517,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'gptsovits_2', computed from 262 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.094, Kids 2.631, Noisy 2.881, Wild 2.636. Identity (ttsds/systems 'gptsovits_2'): lj1995/GPT-SoVITS gsv-v2final-pretrained (s2G2333k.pth + s1bert25hz-5kh-longer-epoch=12-step=369668.ckpt). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 262,
    "per_domain": {
      "Clean": 3.09434,
      "Kids": 2.630769,
      "Noisy": 2.880597,
      "Wild": 2.636364
    },
    "source_model_label": "gptsovits_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "gpt-sovits-v2",
    "bench": "ttsds2-listening-smos",
    "score": 0.704309,
    "raw_value": 3.817237,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'gptsovits_2', computed from 154 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.667, Kids 3.390, Noisy 4.235, Wild 3.977. Identity (ttsds/systems 'gptsovits_2'): lj1995/GPT-SoVITS gsv-v2final-pretrained (s2G2333k.pth + s1bert25hz-5kh-longer-epoch=12-step=369668.ckpt). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 154,
    "per_domain": {
      "Clean": 3.666667,
      "Kids": 3.390244,
      "Noisy": 4.235294,
      "Wild": 3.976744
    },
    "source_model_label": "gptsovits_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "hierspeechpp-v1-1",
    "bench": "ttsds2-listening-mos",
    "score": 0.519454,
    "raw_value": 3.077818,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'hierspeechpp_1_1', computed from 202 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.628, Kids 2.653, Noisy 3.121, Wild 2.909. Identity (ttsds/systems 'hierspeechpp_1_1'): HierSpeech++ Large v1.1 (eng/kor, epoch 200): the systems repo downloads Drive file 1xMfhg4qeehGO0RN-zxq-hAnW-omXmpdq, whose Drive title is 'hierspeechpp_v1.1_ckpt.pth', with the TTV LibriTTS-960 text-to-vec model. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 202,
    "per_domain": {
      "Clean": 3.627907,
      "Kids": 2.653061,
      "Noisy": 3.121212,
      "Wild": 2.909091
    },
    "source_model_label": "hierspeechpp_1_1",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "hierspeechpp-v1-1",
    "bench": "ttsds2-listening-smos",
    "score": 0.620653,
    "raw_value": 3.482612,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'hierspeechpp_1_1', computed from 144 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.038, Kids 3.463, Noisy 3.714, Wild 2.714. Identity (ttsds/systems 'hierspeechpp_1_1'): HierSpeech++ Large v1.1 (eng/kor, epoch 200): the systems repo downloads Drive file 1xMfhg4qeehGO0RN-zxq-hAnW-omXmpdq, whose Drive title is 'hierspeechpp_v1.1_ckpt.pth', with the TTV LibriTTS-960 text-to-vec model. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 144,
    "per_domain": {
      "Clean": 4.038462,
      "Kids": 3.463415,
      "Noisy": 3.714286,
      "Wild": 2.714286
    },
    "source_model_label": "hierspeechpp_1_1",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "maskgct",
    "bench": "ttsds2-listening-mos",
    "score": 0.569966,
    "raw_value": 3.279863,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'maskgct', computed from 251 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.898, Kids 3.258, Noisy 3.269, Wild 3.695. Identity (ttsds/systems 'amphion_maskgct'): amphion/MaskGCT (semantic codec, acoustic codec, T2S and S2A models). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 251,
    "per_domain": {
      "Clean": 2.898305,
      "Kids": 3.257576,
      "Noisy": 3.268657,
      "Wild": 3.694915
    },
    "source_model_label": "maskgct",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "maskgct",
    "bench": "ttsds2-listening-smos",
    "score": 0.847764,
    "raw_value": 4.391058,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'maskgct', computed from 160 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.429, Kids 4.333, Noisy 4.500, Wild 4.302. Identity (ttsds/systems 'amphion_maskgct'): amphion/MaskGCT (semantic codec, acoustic codec, T2S and S2A models). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 160,
    "per_domain": {
      "Clean": 4.428571,
      "Kids": 4.333333,
      "Noisy": 4.5,
      "Wild": 4.302326
    },
    "source_model_label": "maskgct",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "ttsds2-listening-mos",
    "score": 0.373199,
    "raw_value": 2.492794,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'metavoice', computed from 269 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.333, Kids 2.281, Noisy 2.318, Wild 3.039. Identity (ttsds/systems 'metavoice'): metavoiceio/metavoice-1B-v0.1. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 269,
    "per_domain": {
      "Clean": 2.333333,
      "Kids": 2.280702,
      "Noisy": 2.318182,
      "Wild": 3.038961
    },
    "source_model_label": "metavoice",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "ttsds2-listening-smos",
    "score": 0.295703,
    "raw_value": 2.182812,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'metavoice', computed from 150 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 2.143, Kids 2.000, Noisy 2.531, Wild 2.057. Identity (ttsds/systems 'metavoice'): metavoiceio/metavoice-1B-v0.1. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 150,
    "per_domain": {
      "Clean": 2.142857,
      "Kids": 2,
      "Noisy": 2.53125,
      "Wild": 2.057143
    },
    "source_model_label": "metavoice",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-naturalspeech2",
    "bench": "ttsds2-listening-mos",
    "score": 0.261263,
    "raw_value": 2.045053,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'naturalspeech2', computed from 237 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 1.792, Kids 1.954, Noisy 2.260, Wild 2.174. Identity (ttsds/systems 'amphion_naturalspeech2'): amphion/naturalspeech2_libritts (Amphion reproduction). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 237,
    "per_domain": {
      "Clean": 1.792453,
      "Kids": 1.953846,
      "Noisy": 2.26,
      "Wild": 2.173913
    },
    "source_model_label": "naturalspeech2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-naturalspeech2",
    "bench": "ttsds2-listening-smos",
    "score": 0.264541,
    "raw_value": 2.058164,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'naturalspeech2', computed from 168 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 2.143, Kids 1.902, Noisy 2.415, Wild 1.773. Identity (ttsds/systems 'amphion_naturalspeech2'): amphion/naturalspeech2_libritts (Amphion reproduction). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 168,
    "per_domain": {
      "Clean": 2.142857,
      "Kids": 1.902439,
      "Noisy": 2.414634,
      "Wild": 1.772727
    },
    "source_model_label": "naturalspeech2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "openvoice-v2",
    "bench": "ttsds2-listening-mos",
    "score": 0.48027,
    "raw_value": 2.921079,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'openvoice_2', computed from 262 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.956, Kids 3.095, Noisy 2.976, Wild 2.658. Identity (ttsds/systems 'openvoice_2'): myshell-ai/OpenVoiceV2 tone-colour converter over MeloTTS base speakers. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 262,
    "per_domain": {
      "Clean": 2.955882,
      "Kids": 3.094595,
      "Noisy": 2.97561,
      "Wild": 2.658228
    },
    "source_model_label": "openvoice_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "openvoice-v2",
    "bench": "ttsds2-listening-smos",
    "score": 0.396341,
    "raw_value": 2.585366,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'openvoice_2', computed from 167 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.634, Kids 2.171, Noisy 2.537, Wild 2.000. Identity (ttsds/systems 'openvoice_2'): myshell-ai/OpenVoiceV2 tone-colour converter over MeloTTS base speakers. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 167,
    "per_domain": {
      "Clean": 3.634146,
      "Kids": 2.170732,
      "Noisy": 2.536585,
      "Wild": 2
    },
    "source_model_label": "openvoice_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "parler-tts-large-v1",
    "bench": "ttsds2-listening-mos",
    "score": 0.347544,
    "raw_value": 2.390177,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'parlertts_large_1_0', computed from 201 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.463, Kids 2.469, Noisy 2.263, Wild 2.365. Identity (ttsds/systems 'parlertts_large_1_0'): parler-tts/parler-tts-large-v1; the paper notes a codebase modification to allow voice cloning, which could degrade performance. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 201,
    "per_domain": {
      "Clean": 2.463415,
      "Kids": 2.46875,
      "Noisy": 2.263158,
      "Wild": 2.365385
    },
    "source_model_label": "parlertts_large_1_0",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "parler-tts-large-v1",
    "bench": "ttsds2-listening-smos",
    "score": 0.482108,
    "raw_value": 2.928431,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'parlertts_large_1_0', computed from 202 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.098, Kids 2.592, Noisy 3.620, Wild 2.404. Identity (ttsds/systems 'parlertts_large_1_0'): parler-tts/parler-tts-large-v1; the paper notes a codebase modification to allow voice cloning, which could degrade performance. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 202,
    "per_domain": {
      "Clean": 3.098039,
      "Kids": 2.591837,
      "Noisy": 3.62,
      "Wild": 2.403846
    },
    "source_model_label": "parlertts_large_1_0",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "pheme-300m",
    "bench": "ttsds2-listening-mos",
    "score": 0.496961,
    "raw_value": 2.987842,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'pheme', computed from 236 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.478, Kids 2.771, Noisy 2.947, Wild 2.756. Identity (ttsds/systems 'pheme'): PolyAI/pheme with fnlp/SpeechTokenizer. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 236,
    "per_domain": {
      "Clean": 3.477612,
      "Kids": 2.770833,
      "Noisy": 2.947368,
      "Wild": 2.755556
    },
    "source_model_label": "pheme",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "pheme-300m",
    "bench": "ttsds2-listening-smos",
    "score": 0.58842,
    "raw_value": 3.353679,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'pheme', computed from 151 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.314, Kids 2.634, Noisy 3.878, Wild 3.588. Identity (ttsds/systems 'pheme'): PolyAI/pheme with fnlp/SpeechTokenizer. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 151,
    "per_domain": {
      "Clean": 3.314286,
      "Kids": 2.634146,
      "Noisy": 3.878049,
      "Wild": 3.588235
    },
    "source_model_label": "pheme",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "speecht5-tts",
    "bench": "ttsds2-listening-mos",
    "score": 0.244398,
    "raw_value": 1.97759,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'speecht5', computed from 219 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.060, Kids 2.105, Noisy 1.745, Wild 2.000. Identity (ttsds/systems 'speecht5'): microsoft/speecht5_tts + microsoft/speecht5_hifigan, speaker x-vector from speechbrain/spkrec-xvect-voxceleb. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 219,
    "per_domain": {
      "Clean": 2.06,
      "Kids": 2.105263,
      "Noisy": 1.745098,
      "Wild": 2
    },
    "source_model_label": "speecht5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "speecht5-tts",
    "bench": "ttsds2-listening-smos",
    "score": 0.407469,
    "raw_value": 2.629875,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'speecht5', computed from 150 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 2.786, Kids 3.333, Noisy 2.610, Wild 1.791. Identity (ttsds/systems 'speecht5'): microsoft/speecht5_tts + microsoft/speecht5_hifigan, speaker x-vector from speechbrain/spkrec-xvect-voxceleb. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 150,
    "per_domain": {
      "Clean": 2.785714,
      "Kids": 3.333333,
      "Noisy": 2.609756,
      "Wild": 1.790698
    },
    "source_model_label": "speecht5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "ttsds2-listening-mos",
    "score": 0.502721,
    "raw_value": 3.010884,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'styletts2', computed from 245 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.329, Kids 2.591, Noisy 3.431, Wild 2.692. Identity (ttsds/systems 'styletts2'): yl4579/StyleTTS2-LibriTTS. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 245,
    "per_domain": {
      "Clean": 3.328947,
      "Kids": 2.590909,
      "Noisy": 3.431373,
      "Wild": 2.692308
    },
    "source_model_label": "styletts2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "ttsds2-listening-smos",
    "score": 0.483012,
    "raw_value": 2.932049,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'styletts2', computed from 169 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.048, Kids 1.902, Noisy 3.119, Wild 2.659. Identity (ttsds/systems 'styletts2'): yl4579/StyleTTS2-LibriTTS. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 169,
    "per_domain": {
      "Clean": 4.047619,
      "Kids": 1.902439,
      "Noisy": 3.119048,
      "Wild": 2.659091
    },
    "source_model_label": "styletts2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "ttsds2-listening-mos",
    "score": 0.555496,
    "raw_value": 3.221984,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'tortoise', computed from 228 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.383, Kids 3.070, Noisy 3.500, Wild 2.934. Identity (ttsds/systems 'tortoise'): jbetker/tortoise-tts-v2. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 228,
    "per_domain": {
      "Clean": 3.383333,
      "Kids": 3.070175,
      "Noisy": 3.5,
      "Wild": 2.934426
    },
    "source_model_label": "tortoise",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "ttsds2-listening-smos",
    "score": 0.431441,
    "raw_value": 2.725766,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'tortoise', computed from 170 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.295, Kids 2.171, Noisy 2.786, Wild 2.651. Identity (ttsds/systems 'tortoise'): jbetker/tortoise-tts-v2. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 170,
    "per_domain": {
      "Clean": 3.295455,
      "Kids": 2.170732,
      "Noisy": 2.785714,
      "Wild": 2.651163
    },
    "source_model_label": "tortoise",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-valle-v2",
    "bench": "ttsds2-listening-mos",
    "score": 0.473988,
    "raw_value": 2.895951,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'valle_v2', computed from 237 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 2.836, Kids 2.810, Noisy 3.051, Wild 2.887. Identity (ttsds/systems 'amphion_valle'): amphion/valle (valle_ar_mls_196000.bin + valle_nar_mls_164000.bin; Amphion VALL-E v2 recipe reproduction). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 237,
    "per_domain": {
      "Clean": 2.835821,
      "Kids": 2.810345,
      "Noisy": 3.050847,
      "Wild": 2.886792
    },
    "source_model_label": "valle_v2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-valle-v2",
    "bench": "ttsds2-listening-smos",
    "score": 0.608457,
    "raw_value": 3.43383,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'valle_v2', computed from 162 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.136, Kids 2.756, Noisy 3.500, Wild 3.343. Identity (ttsds/systems 'amphion_valle'): amphion/valle (valle_ar_mls_196000.bin + valle_nar_mls_164000.bin; Amphion VALL-E v2 recipe reproduction). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 162,
    "per_domain": {
      "Clean": 4.136364,
      "Kids": 2.756098,
      "Noisy": 3.5,
      "Wild": 3.342857
    },
    "source_model_label": "valle_v2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-vevo",
    "bench": "ttsds2-listening-mos",
    "score": 0.590829,
    "raw_value": 3.363314,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'vevo', computed from 238 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.373, Kids 3.224, Noisy 3.071, Wild 3.785. Identity (ttsds/systems 'amphion_vevo'): amphion/Vevo: vq8192 content-style tokenizer, PhoneToVq8192 AR model, Vq8192ToMels flow matching, Vocoder (Vevo 1.0 TTS pipeline, not Vevo1.5). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 238,
    "per_domain": {
      "Clean": 3.372881,
      "Kids": 3.224138,
      "Noisy": 3.071429,
      "Wild": 3.78481
    },
    "source_model_label": "vevo",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-vevo",
    "bench": "ttsds2-listening-smos",
    "score": 0.753467,
    "raw_value": 4.013868,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'vevo', computed from 141 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 4.048, Kids 3.906, Noisy 4.455, Wild 3.647. Identity (ttsds/systems 'amphion_vevo'): amphion/Vevo: vq8192 content-style tokenizer, PhoneToVq8192 AR model, Vq8192ToMels flow matching, Vocoder (Vevo 1.0 TTS pipeline, not Vevo1.5). Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 141,
    "per_domain": {
      "Clean": 4.047619,
      "Kids": 3.90625,
      "Noisy": 4.454545,
      "Wild": 3.647059
    },
    "source_model_label": "vevo",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "whisperspeech-v1-95-medium",
    "bench": "ttsds2-listening-mos",
    "score": 0.529277,
    "raw_value": 3.117109,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean MOS (naturalness) for system 'whisperspeech_medium', computed from 234 individual ratings in subjective_results.csv (rating_type=mos). Per-domain means: Clean 3.120, Kids 3.215, Noisy 2.983, Wild 3.150. Identity (ttsds/systems 'whisperspeech'): collabora/whisperspeech 'medium' pipeline: t2s-v1.95-medium-7lang.model + s2a-v1.95-medium-7lang.model. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 234,
    "per_domain": {
      "Clean": 3.12,
      "Kids": 3.215385,
      "Noisy": 2.983051,
      "Wild": 3.15
    },
    "source_model_label": "whisperspeech_medium",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "whisperspeech-v1-95-medium",
    "bench": "ttsds2-listening-smos",
    "score": 0.421218,
    "raw_value": 2.684874,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv",
    "source_id": "speech-ttsds2-listening-mit",
    "by": "independent",
    "date": "2025-05-11",
    "date_basis": "commit date of the pinned listening_test revision; ratings collected for the TTSDS2 study (arXiv:2506.19441, June 2025)",
    "note": "Mean over the four domains (Clean, Noisy, Wild, Kids) of the per-domain mean SMOS (speaker similarity) for system 'whisperspeech_medium', computed from 169 individual ratings in subjective_results.csv (rating_type=smos). Per-domain means: Clean 3.071, Kids 2.195, Noisy 2.905, Wild 2.568. Identity (ttsds/systems 'whisperspeech'): collabora/whisperspeech 'medium' pipeline: t2s-v1.95-medium-7lang.model + s2a-v1.95-medium-7lang.model. Checkpoints: latest before the 2025-01-01 data cut-off.",
    "sample_count": 169,
    "per_domain": {
      "Clean": 3.071429,
      "Kids": 2.195122,
      "Noisy": 2.904762,
      "Wild": 2.568182
    },
    "source_model_label": "whisperspeech_medium",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8948535983,
    "raw_value": 89.48535983182133,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='e2', score_mean=89.48535983182133 (= unweighted mean of SPEAKER 88.86, PROSODY 92.85, INTELLIGIBILITY 80.37, GENERIC 95.87). Identity (ttsds/systems 'e2'): SWivid/E2-TTS E2TTS_Base/model_1200000.safetensors (F5-TTS authors' E2 TTS reproduction; Microsoft's original is unreleased).",
    "components": {
      "SPEAKER": 88.8562980651073,
      "PROSODY": 92.85013379730026,
      "INTELLIGIBILITY": 80.36776179983642,
      "GENERIC": 95.86724566504138
    },
    "source_model_label": "e2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8955488839,
    "raw_value": 89.55488839041308,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='f5', score_mean=89.55488839041308 (= unweighted mean of SPEAKER 87.69, PROSODY 92.14, INTELLIGIBILITY 82.31, GENERIC 96.09). Identity (ttsds/systems 'f5'): SWivid/F5-TTS F5TTS_Base/model_1200000.safetensors (F5-TTS v0 Base; not F5TTS_v1_Base, released 2025-03 after the 2025-01-01 cut-off).",
    "components": {
      "SPEAKER": 87.6862477936837,
      "PROSODY": 92.13501290548982,
      "INTELLIGIBILITY": 82.31061481910066,
      "GENERIC": 96.08767804337815
    },
    "source_model_label": "f5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "maskgct",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.9031293301,
    "raw_value": 90.31293300506799,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='maskgct', score_mean=90.31293300506799 (= unweighted mean of SPEAKER 87.44, PROSODY 94.39, INTELLIGIBILITY 82.19, GENERIC 97.23). Identity (ttsds/systems 'amphion_maskgct'): amphion/MaskGCT (semantic codec, acoustic codec, T2S and S2A models).",
    "components": {
      "SPEAKER": 87.44483077461761,
      "PROSODY": 94.39076193835619,
      "INTELLIGIBILITY": 82.18619558770749,
      "GENERIC": 97.22994371959066
    },
    "source_model_label": "maskgct",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "fish-speech-1-5",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8681657044,
    "raw_value": 86.81657044118326,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='fishspeech_1_5', score_mean=86.81657044118326 (= unweighted mean of SPEAKER 82.98, PROSODY 89.79, INTELLIGIBILITY 80.54, GENERIC 93.96). Identity (ttsds/systems 'fishspeech_1_5'): fishaudio/fish-speech-1.5 (firefly-gan-vq-fsq-8x1024-21hz decoder).",
    "components": {
      "SPEAKER": 82.97512576940375,
      "PROSODY": 89.79317181270532,
      "INTELLIGIBILITY": 80.54217769483367,
      "GENERIC": 93.9558064877903
    },
    "source_model_label": "fishspeech_1_5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8776211795,
    "raw_value": 87.76211795437726,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='tortoise', score_mean=87.76211795437726 (= unweighted mean of SPEAKER 81.92, PROSODY 91.77, INTELLIGIBILITY 81.58, GENERIC 95.79). Identity (ttsds/systems 'tortoise'): jbetker/tortoise-tts-v2.",
    "components": {
      "SPEAKER": 81.91627047230574,
      "PROSODY": 91.76898157426044,
      "INTELLIGIBILITY": 81.5775680949175,
      "GENERIC": 95.78565167602534
    },
    "source_model_label": "tortoise",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/tortoise/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-vevo",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8652125469,
    "raw_value": 86.52125469397788,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='vevo', score_mean=86.52125469397788 (= unweighted mean of SPEAKER 81.16, PROSODY 89.47, INTELLIGIBILITY 80.42, GENERIC 95.03). Identity (ttsds/systems 'amphion_vevo'): amphion/Vevo: vq8192 content-style tokenizer, PhoneToVq8192 AR model, Vq8192ToMels flow matching, Vocoder (Vevo 1.0 TTS pipeline, not Vevo1.5).",
    "components": {
      "SPEAKER": 81.1607529605971,
      "PROSODY": 89.47413243945226,
      "INTELLIGIBILITY": 80.4207003818099,
      "GENERIC": 95.02943299405229
    },
    "source_model_label": "vevo",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "gpt-sovits-v2",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.872206503,
    "raw_value": 87.22065030467198,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='gptsovits_2', score_mean=87.22065030467198 (= unweighted mean of SPEAKER 79.81, PROSODY 93.91, INTELLIGIBILITY 80.83, GENERIC 94.34). Identity (ttsds/systems 'gptsovits_2'): lj1995/GPT-SoVITS gsv-v2final-pretrained (s2G2333k.pth + s1bert25hz-5kh-longer-epoch=12-step=369668.ckpt).",
    "components": {
      "SPEAKER": 79.81172981574952,
      "PROSODY": 93.90749648624055,
      "INTELLIGIBILITY": 80.82711032486128,
      "GENERIC": 94.3362645918366
    },
    "source_model_label": "gptsovits_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/gptsovits_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "hierspeechpp-v1-1",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8633754697,
    "raw_value": 86.33754697113233,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='hierspeechpp_1_1', score_mean=86.33754697113233 (= unweighted mean of SPEAKER 78.37, PROSODY 91.54, INTELLIGIBILITY 81.23, GENERIC 94.22). Identity (ttsds/systems 'hierspeechpp_1_1'): HierSpeech++ Large v1.1 (eng/kor, epoch 200): the systems repo downloads Drive file 1xMfhg4qeehGO0RN-zxq-hAnW-omXmpdq, whose Drive title is 'hierspeechpp_v1.1_ckpt.pth', with the TTV LibriTTS-960 text-to-vec model.",
    "components": {
      "SPEAKER": 78.36553593055356,
      "PROSODY": 91.53552719855679,
      "INTELLIGIBILITY": 81.23379716441127,
      "GENERIC": 94.21532759100768
    },
    "source_model_label": "hierspeechpp_1_1",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/hierspeechpp_1_1/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "openvoice-v2",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8245606306,
    "raw_value": 82.4560630630403,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='openvoice_2', score_mean=82.4560630630403 (= unweighted mean of SPEAKER 77.06, PROSODY 86.23, INTELLIGIBILITY 73.87, GENERIC 92.68). Identity (ttsds/systems 'openvoice_2'): myshell-ai/OpenVoiceV2 tone-colour converter over MeloTTS base speakers.",
    "components": {
      "SPEAKER": 77.05573946939313,
      "PROSODY": 86.22545672217755,
      "INTELLIGIBILITY": 73.86638599248505,
      "GENERIC": 92.67667006810545
    },
    "source_model_label": "openvoice_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "whisperspeech-v1-95-medium",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8618720288,
    "raw_value": 86.18720288417515,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='whisperspeech_medium', score_mean=86.18720288417515 (= unweighted mean of SPEAKER 76.78, PROSODY 92.53, INTELLIGIBILITY 79.16, GENERIC 96.28). Identity (ttsds/systems 'whisperspeech'): collabora/whisperspeech 'medium' pipeline: t2s-v1.95-medium-7lang.model + s2a-v1.95-medium-7lang.model.",
    "components": {
      "SPEAKER": 76.77998953676396,
      "PROSODY": 92.53311355423538,
      "INTELLIGIBILITY": 79.15649842944498,
      "GENERIC": 96.27921001625629
    },
    "source_model_label": "whisperspeech_medium",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/whisperspeech/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "pheme-300m",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8705166213,
    "raw_value": 87.05166212848297,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='pheme', score_mean=87.05166212848297 (= unweighted mean of SPEAKER 76.61, PROSODY 94.05, INTELLIGIBILITY 81.58, GENERIC 95.96). Identity (ttsds/systems 'pheme'): PolyAI/pheme with fnlp/SpeechTokenizer.",
    "components": {
      "SPEAKER": 76.61284658424431,
      "PROSODY": 94.05166764323752,
      "INTELLIGIBILITY": 81.58050912889433,
      "GENERIC": 95.9616251575557
    },
    "source_model_label": "pheme",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/pheme/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "metavoice-1b-v0-1",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8633382335,
    "raw_value": 86.33382334791887,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='metavoice', score_mean=86.33382334791887 (= unweighted mean of SPEAKER 76.54, PROSODY 91.49, INTELLIGIBILITY 81.35, GENERIC 95.96). Identity (ttsds/systems 'metavoice'): metavoiceio/metavoice-1B-v0.1.",
    "components": {
      "SPEAKER": 76.53748220641,
      "PROSODY": 91.49185810729918,
      "INTELLIGIBILITY": 81.34949035723969,
      "GENERIC": 95.95646272072656
    },
    "source_model_label": "metavoice",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/metavoice/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8467947913,
    "raw_value": 84.67947913027686,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='styletts2', score_mean=84.67947913027686 (= unweighted mean of SPEAKER 76.49, PROSODY 89.09, INTELLIGIBILITY 80.94, GENERIC 92.19). Identity (ttsds/systems 'styletts2'): yl4579/StyleTTS2-LibriTTS.",
    "components": {
      "SPEAKER": 76.48509765049275,
      "PROSODY": 89.0945657529155,
      "INTELLIGIBILITY": 80.9440912472866,
      "GENERIC": 92.19416187041259
    },
    "source_model_label": "styletts2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/styletts2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "bark",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8222711933,
    "raw_value": 82.2271193258945,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='bark', score_mean=82.2271193258945 (= unweighted mean of SPEAKER 74.72, PROSODY 81.02, INTELLIGIBILITY 79.21, GENERIC 93.96). Identity (ttsds/systems 'bark'): suno/bark full-size models (text_2/coarse_2/fine_2; bark_small is a separate, unrated system) with GitMylo HuBERT-quantizer voice-cloning prompts.",
    "components": {
      "SPEAKER": 74.71840994381664,
      "PROSODY": 81.01770748629633,
      "INTELLIGIBILITY": 79.21223518996585,
      "GENERIC": 93.96012468349919
    },
    "source_model_label": "bark",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/bark/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-naturalspeech2",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8123354691,
    "raw_value": 81.23354690683949,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='naturalspeech2', score_mean=81.23354690683949 (= unweighted mean of SPEAKER 71.05, PROSODY 87.10, INTELLIGIBILITY 74.90, GENERIC 91.89). Identity (ttsds/systems 'amphion_naturalspeech2'): amphion/naturalspeech2_libritts (Amphion reproduction).",
    "components": {
      "SPEAKER": 71.04830079818787,
      "PROSODY": 87.09988828195812,
      "INTELLIGIBILITY": 74.89996066252193,
      "GENERIC": 91.88603788469004
    },
    "source_model_label": "naturalspeech2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_naturalspeech2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-valle-v2",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8210814061,
    "raw_value": 82.10814060846413,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='valle_v2', score_mean=82.10814060846413 (= unweighted mean of SPEAKER 70.13, PROSODY 87.73, INTELLIGIBILITY 77.65, GENERIC 92.92). Identity (ttsds/systems 'amphion_valle'): amphion/valle (valle_ar_mls_196000.bin + valle_nar_mls_164000.bin; Amphion VALL-E v2 recipe reproduction).",
    "components": {
      "SPEAKER": 70.12778524050978,
      "PROSODY": 87.72707669571423,
      "INTELLIGIBILITY": 77.65333630317579,
      "GENERIC": 92.92436419445671
    },
    "source_model_label": "valle_v2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_valle/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "speecht5-tts",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.8257772823,
    "raw_value": 82.57772823014776,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='speecht5', score_mean=82.57772823014776 (= unweighted mean of SPEAKER 69.85, PROSODY 90.46, INTELLIGIBILITY 76.67, GENERIC 93.33). Identity (ttsds/systems 'speecht5'): microsoft/speecht5_tts + microsoft/speecht5_hifigan, speaker x-vector from speechbrain/spkrec-xvect-voxceleb.",
    "components": {
      "SPEAKER": 69.8503499590561,
      "PROSODY": 90.45918079223647,
      "INTELLIGIBILITY": 76.66932080309022,
      "GENERIC": 93.33206136620824
    },
    "source_model_label": "speecht5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/speecht5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "parler-tts-large-v1",
    "bench": "ttsds2-multilingual-en-overall",
    "score": 0.79514715,
    "raw_value": 79.51471500096682,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "en_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='parlertts_large_1_0', score_mean=79.51471500096682 (= unweighted mean of SPEAKER 68.45, PROSODY 84.37, INTELLIGIBILITY 75.68, GENERIC 89.56). Identity (ttsds/systems 'parlertts_large_1_0'): parler-tts/parler-tts-large-v1; the paper notes a codebase modification to allow voice cloning, which could degrade performance.",
    "components": {
      "SPEAKER": 68.44573075969224,
      "PROSODY": 84.36881743411709,
      "INTELLIGIBILITY": 75.68466134051108,
      "GENERIC": 89.55965046954687
    },
    "source_model_label": "parlertts_large_1_0",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/parlertts_large_1_0/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.897949141,
    "raw_value": 89.79491410212353,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='e2', score_mean=89.79491410212353 (= unweighted mean of SPEAKER 89.58, PROSODY 91.03, INTELLIGIBILITY 82.12, GENERIC 96.45). Identity (ttsds/systems 'e2'): SWivid/E2-TTS E2TTS_Base/model_1200000.safetensors (F5-TTS authors' E2 TTS reproduction; Microsoft's original is unreleased).",
    "components": {
      "SPEAKER": 89.58367279725168,
      "PROSODY": 91.03201036604833,
      "INTELLIGIBILITY": 82.11664460139536,
      "GENERIC": 96.44732864379876
    },
    "source_model_label": "e2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/e2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.8939981955,
    "raw_value": 89.39981954652254,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='f5', score_mean=89.39981954652254 (= unweighted mean of SPEAKER 87.86, PROSODY 91.07, INTELLIGIBILITY 83.20, GENERIC 95.47). Identity (ttsds/systems 'f5'): SWivid/F5-TTS F5TTS_Base/model_1200000.safetensors (F5-TTS v0 Base; not F5TTS_v1_Base, released 2025-03 after the 2025-01-01 cut-off).",
    "components": {
      "SPEAKER": 87.85828097165742,
      "PROSODY": 91.07308306967067,
      "INTELLIGIBILITY": 83.1950330698532,
      "GENERIC": 95.47288107490886
    },
    "source_model_label": "f5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/f5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "amphion-vevo",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.868101065,
    "raw_value": 86.81010649820816,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='vevo', score_mean=86.81010649820816 (= unweighted mean of SPEAKER 87.01, PROSODY 86.07, INTELLIGIBILITY 77.74, GENERIC 96.42). Identity (ttsds/systems 'amphion_vevo'): amphion/Vevo: vq8192 content-style tokenizer, PhoneToVq8192 AR model, Vq8192ToMels flow matching, Vocoder (Vevo 1.0 TTS pipeline, not Vevo1.5).",
    "components": {
      "SPEAKER": 87.00800880071473,
      "PROSODY": 86.07058668057257,
      "INTELLIGIBILITY": 77.73756955694343,
      "GENERIC": 96.4242609546019
    },
    "source_model_label": "vevo",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_vevo/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "maskgct",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.8908341142,
    "raw_value": 89.08341142108681,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='maskgct', score_mean=89.08341142108681 (= unweighted mean of SPEAKER 85.42, PROSODY 92.77, INTELLIGIBILITY 82.96, GENERIC 95.19). Identity (ttsds/systems 'amphion_maskgct'): amphion/MaskGCT (semantic codec, acoustic codec, T2S and S2A models).",
    "components": {
      "SPEAKER": 85.41670126898009,
      "PROSODY": 92.76955277299928,
      "INTELLIGIBILITY": 82.9576213475855,
      "GENERIC": 95.18977029478235
    },
    "source_model_label": "maskgct",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/amphion_maskgct/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "fish-speech-1-5",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.8056850546,
    "raw_value": 80.5685054550879,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='fishspeech_1_5', score_mean=80.5685054550879 (= unweighted mean of SPEAKER 77.47, PROSODY 76.29, INTELLIGIBILITY 77.23, GENERIC 91.29). Identity (ttsds/systems 'fishspeech_1_5'): fishaudio/fish-speech-1.5 (firefly-gan-vq-fsq-8x1024-21hz decoder).",
    "components": {
      "SPEAKER": 77.46536922585506,
      "PROSODY": 76.28991003735194,
      "INTELLIGIBILITY": 77.22628121079507,
      "GENERIC": 91.29246134634953
    },
    "source_model_label": "fishspeech_1_5",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/fishspeech_1_5/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "openvoice-v2",
    "bench": "ttsds2-multilingual-zh-overall",
    "score": 0.8197154379,
    "raw_value": 81.97154379095909,
    "scale_divisor": 100,
    "source": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv",
    "source_id": "speech-ttsds2-multilingual-mit",
    "by": "independent",
    "date": "2025-05-15",
    "date_basis": "commit date of the pinned leaderboard_data revision",
    "note": "zh_aggregated_multilingual.csv row benchmark_category=OVERALL, dataset='openvoice_2', score_mean=81.97154379095909 (= unweighted mean of SPEAKER 69.72, PROSODY 90.61, INTELLIGIBILITY 75.88, GENERIC 91.68). Identity (ttsds/systems 'openvoice_2'): myshell-ai/OpenVoiceV2 tone-colour converter over MeloTTS base speakers.",
    "components": {
      "SPEAKER": 69.71774835561399,
      "PROSODY": 90.6090878185371,
      "INTELLIGIBILITY": 75.87987358537237,
      "GENERIC": 91.67946540431292
    },
    "source_model_label": "openvoice_2",
    "identity_sources": [
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/download.sh",
      "https://github.com/ttsds/systems/blob/c5836668e3e0f1c1655732d462a06a1814098851/models/openvoice_2/predict.py",
      "https://arxiv.org/abs/2506.19441v2"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0359,
    "raw_value": 3.59,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'CosyVoice', column WER(%) = 3.59; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.66,
    "raw_value": 0.66,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'CosyVoice', column SIM-o = 0.66; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0339,
    "raw_value": 3.39,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'CosyVoice', column WER(%) = 3.39; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.64,
    "raw_value": 0.64,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'CosyVoice', column SIM-o = 0.64; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.031,
    "raw_value": 3.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'CosyVoice', column WER(%) = 3.10 (Chinese character error rate under the paper's WER heading); three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.75,
    "raw_value": 0.75,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'CosyVoice', column SIM-o = 0.75; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint huggingface.co/model-scope/CosyVoice-300M.",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://huggingface.co/model-scope/CosyVoice-300M",
      "https://huggingface.co/FunAudioLLM/CosyVoice-300M"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0269,
    "raw_value": 2.69,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'FireRedTTS', column WER(%) = 2.69; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.47,
    "raw_value": 0.47,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'FireRedTTS', column SIM-o = 0.47; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0382,
    "raw_value": 3.82,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'FireRedTTS', column WER(%) = 3.82; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.46,
    "raw_value": 0.46,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'FireRedTTS', column SIM-o = 0.46; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0151,
    "raw_value": 1.51,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'FireRedTTS', column WER(%) = 1.51 (Chinese character error rate under the paper's WER heading); three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.63,
    "raw_value": 0.63,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "independent",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'FireRedTTS', column SIM-o = 0.63; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Appendix A: official code and pre-trained checkpoint github.com/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0295,
    "raw_value": 2.95,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'E2 TTS (32 NFE)', column WER(%) = 2.95; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.69,
    "raw_value": 0.69,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'E2 TTS (32 NFE)', column SIM-o = 0.69; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0219,
    "raw_value": 2.19,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'E2 TTS (32 NFE)', column WER(%) = 2.19; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.71,
    "raw_value": 0.71,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'E2 TTS (32 NFE)', column SIM-o = 0.71; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0197,
    "raw_value": 1.97,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'E2 TTS (32 NFE)', column WER(%) = 1.97 (Chinese character error rate under the paper's WER heading); three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.73,
    "raw_value": 0.73,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'E2 TTS (32 NFE)', column SIM-o = 0.73; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' own reproduced 333M multilingual E2 TTS trained on Emilia to 1.2M updates, released as SWivid/E2-TTS E2TTS_Base (Microsoft's original E2 TTS is unreleased).",
    "source_model_label": "E2 TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0242,
    "raw_value": 2.42,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'F5-TTS (32 NFE)', column WER(%) = 2.42; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.66,
    "raw_value": 0.66,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 1, section 'LibriSpeech-PC test-clean', row 'F5-TTS (32 NFE)', column SIM-o = 0.66; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0183,
    "raw_value": 1.83,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'F5-TTS (32 NFE)', column WER(%) = 1.83; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.67,
    "raw_value": 0.67,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-en', row 'F5-TTS (32 NFE)', column SIM-o = 0.67; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0156,
    "raw_value": 1.56,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'F5-TTS (32 NFE)', column WER(%) = 1.56 (Chinese character error rate under the paper's WER heading); three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.76,
    "raw_value": 0.76,
    "source": "https://arxiv.org/html/2410.06885v1",
    "source_id": "speech-f5tts-report-ccby",
    "by": "lab",
    "date": "2024-10-09",
    "note": "Table 2, section 'Seed-TTS test-zh', row 'F5-TTS (32 NFE)', column SIM-o = 0.76; three-seed average; Whisper-large-v3 (EN) / Paraformer-zh (ZH) ASR, WavLM-large speaker-verification SIM-o. Authors' F5-TTS base (336M, 1.2M updates, CFG 2, sway -1, 32 NFE, Vocos), released as SWivid/F5-TTS F5TTS_Base; v1 paper (Oct 2024) predates the separate F5TTS_v1_Base (Mar 2025).",
    "source_model_label": "F5-TTS (32 NFE)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0257,
    "raw_value": 2.57,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset, 40 prompts, 1127 utterances), row 'IndexTTS', column WER(%) = 2.57; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.62,
    "raw_value": 0.62,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'IndexTTS', column SIM = 0.62; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "librispeech-pc-test-clean-f5subset-utmos",
    "score": 0.8375,
    "raw_value": 4.35,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'IndexTTS', column UTMOS = 4.35; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0192,
    "raw_value": 1.92,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'IndexTTS', column WER(%) = 1.92; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.61,
    "raw_value": 0.61,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'IndexTTS', column SIM = 0.61; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "indextts",
    "bench": "seed-tts-en-utmos",
    "score": 0.745,
    "raw_value": 3.98,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'IndexTTS', column UTMOS = 3.98; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: official code and pre-trained checkpoint huggingface.co/IndexTeam/Index-TTS (IndexTTS v1).",
    "source_model_label": "IndexTTS",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/Index-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0274,
    "raw_value": 2.74,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset, 40 prompts, 1127 utterances), row 'E2TTS (NFE=32)', column WER(%) = 2.74; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.7,
    "raw_value": 0.7,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'E2TTS (NFE=32)', column SIM = 0.70; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "librispeech-pc-test-clean-f5subset-utmos",
    "score": 0.6175,
    "raw_value": 3.47,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'E2TTS (NFE=32)', column UTMOS = 3.47; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.022,
    "raw_value": 2.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'E2TTS (NFE=32)', column WER(%) = 2.20; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.71,
    "raw_value": 0.71,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'E2TTS (NFE=32)', column SIM = 0.71; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-utmos",
    "score": 0.55,
    "raw_value": 3.2,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'E2TTS (NFE=32)', column UTMOS = 3.20; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Appendix E: unofficial implementation and pre-trained checkpoint huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "librispeech-pc-test-clean-f5subset-whisper-v3-wer",
    "score": 0.0202,
    "raw_value": 2.02,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset, 40 prompts, 1127 utterances), row 'F5TTS-v1 (NFE=32)', column WER(%) = 2.02; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "librispeech-pc-test-clean-f5subset-wavlm-sim",
    "score": 0.68,
    "raw_value": 0.68,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'F5TTS-v1 (NFE=32)', column SIM = 0.68; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "librispeech-pc-test-clean-f5subset-utmos",
    "score": 0.7075,
    "raw_value": 3.83,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'LibriSpeech test-clean' (F5-TTS LibriSpeech-PC subset), row 'F5TTS-v1 (NFE=32)', column UTMOS = 3.83; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0135,
    "raw_value": 1.35,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'F5TTS-v1 (NFE=32)', column WER(%) = 1.35; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.68,
    "raw_value": 0.68,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'F5TTS-v1 (NFE=32)', column SIM = 0.68; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "seed-tts-en-utmos",
    "score": 0.665,
    "raw_value": 3.66,
    "raw_unit": "predicted MOS (UTMOS, 1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 1, section 'Seed-TTS test-en', row 'F5TTS-v1 (NFE=32)', column UTMOS = 3.66; Whisper-large-v3 WER, WavLM-TDNN speaker-verification SIM vs prompt, UTMOS predictor. Row label 'F5TTS-v1'; Appendix E: official code and pre-trained checkpoint huggingface.co/SWivid/F5-TTS (F5TTS_v1_Base).",
    "source_model_label": "F5TTS-v1 (NFE=32)",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "fireredtts-1",
    "bench": "seed-tts-en-human-smos",
    "score": 0.34,
    "raw_value": 2.36,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 2 (subjective, Seed-TTS test-en), row 'FireRedTTS', column SMOS = 2.36 ± 0.58. Human speaker-similarity MOS on 1-5; Human (ground truth) row = 3.07. Appendix E: huggingface.co/FireRedTeam/FireRedTTS.",
    "source_model_label": "FireRedTTS",
    "identity_sources": [
      "https://github.com/FireRedTeam/FireRedTTS",
      "https://huggingface.co/FireRedTeam/FireRedTTS"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-human-smos",
    "score": 0.5175,
    "raw_value": 3.07,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 2 (subjective, Seed-TTS test-en), row 'CosyVoice 2', column SMOS = 3.07 ± 0.21. Human speaker-similarity MOS on 1-5; Human (ground truth) row = 3.07. Appendix E: huggingface.co/FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "e2-tts-base",
    "bench": "seed-tts-en-human-smos",
    "score": 0.5725,
    "raw_value": 3.29,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 2 (subjective, Seed-TTS test-en), row 'E2TTS', column SMOS = 3.29 ± 0.19. Human speaker-similarity MOS on 1-5; Human (ground truth) row = 3.07. Appendix E: huggingface.co/SWivid/E2-TTS.",
    "source_model_label": "E2TTS",
    "identity_sources": [
      "https://huggingface.co/SWivid/E2-TTS"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "seed-tts-en-human-smos",
    "score": 0.52,
    "raw_value": 3.08,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2510.12210v2",
    "source_id": "speech-distar-report-ccby",
    "by": "independent",
    "date": "2025-10-15",
    "note": "Table 2 (subjective, Seed-TTS test-en), row 'F5TTS', column SMOS = 3.08 ± 0.20. Human speaker-similarity MOS on 1-5; Human (ground truth) row = 3.07. Same single F5-TTS baseline as Table 1 'F5TTS-v1'; Appendix E lists one F5-TTS checkpoint (SWivid/F5-TTS).",
    "source_model_label": "F5TTS",
    "identity_sources": [
      "https://huggingface.co/SWivid/F5-TTS",
      "https://github.com/SWivid/F5-TTS"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.03358,
    "raw_value": 3.358,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'CosyVoice 2', column Seed-TTS-en WER = 3.358; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.641,
    "raw_value": 0.641,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'CosyVoice 2', column Seed-TTS-en SIM = 0.641; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.01582,
    "raw_value": 1.582,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'CosyVoice 2', column Seed-TTS-zh CER = 1.582; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.754,
    "raw_value": 0.754,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'CosyVoice 2', column Seed-TTS-zh SIM = 0.754; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.02308,
    "raw_value": 2.308,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'Spark-TTS', column Seed-TTS-en WER = 2.308; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.572,
    "raw_value": 0.572,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'Spark-TTS', column Seed-TTS-en SIM = 0.572; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.01717,
    "raw_value": 1.717,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'Spark-TTS', column Seed-TTS-zh CER = 1.717; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.657,
    "raw_value": 0.657,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'Spark-TTS', column Seed-TTS-zh SIM = 0.657; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.02622,
    "raw_value": 2.622,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'MaskGCT', column Seed-TTS-en WER = 2.622; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.713,
    "raw_value": 0.713,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'MaskGCT', column Seed-TTS-en SIM = 0.713; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.02395,
    "raw_value": 2.395,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'MaskGCT', column Seed-TTS-zh CER = 2.395; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.772,
    "raw_value": 0.772,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'MaskGCT', column Seed-TTS-zh SIM = 0.772; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.03994,
    "raw_value": 3.994,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'LLaSA-8B', column Seed-TTS-en WER = 3.994; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "LLaSA-8B",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.594,
    "raw_value": 0.594,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'LLaSA-8B', column Seed-TTS-en SIM = 0.594; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "LLaSA-8B",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.04214,
    "raw_value": 4.214,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'LLaSA-8B', column Seed-TTS-zh CER = 4.214; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "LLaSA-8B",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.671,
    "raw_value": 0.671,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "independent",
    "date": "2025-07-20",
    "note": "Table 2, row 'LLaSA-8B', column Seed-TTS-zh SIM = 0.671; 'All samples from baseline models were synthesized using the official checkpoints released by the authors'; Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "LLaSA-8B",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "stylettszs-librispeech40-mos-n",
    "score": 0.8075,
    "raw_value": 4.23,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'StyleTTS 2', column MOS-N (CI) = 4.23 (± 0.11); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). yl4579/StyleTTS2-LibriTTS checkpoint (Appendix B-A); 5 style-diffusion steps. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "StyleTTS 2",
    "identity_sources": [
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "stylettszs-librispeech40-mos-s",
    "score": 0.605,
    "raw_value": 3.42,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'StyleTTS 2', column MOS-S (CI) = 3.42 (± 0.09); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). yl4579/StyleTTS2-LibriTTS checkpoint (Appendix B-A); 5 style-diffusion steps. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "StyleTTS 2",
    "identity_sources": [
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "stylettszs-librispeech40-hubert-wer",
    "score": 0.0161,
    "raw_value": 1.61,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'StyleTTS 2', column WER = 1.61; 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). yl4579/StyleTTS2-LibriTTS checkpoint (Appendix B-A); 5 style-diffusion steps. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "StyleTTS 2",
    "identity_sources": [
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ]
  },
  {
    "model": "hierspeechpp-libritts960",
    "bench": "stylettszs-librispeech40-mos-n",
    "score": 0.635,
    "raw_value": 3.54,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'HierSpeech++', column MOS-N (CI) = 3.54 (± 0.12); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). 'LibriTTS-train-960 checkpoint' of github.com/sh-lee-prml/HierSpeechpp (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "HierSpeech++",
    "identity_sources": [
      "https://github.com/sh-lee-prml/HierSpeechpp"
    ]
  },
  {
    "model": "hierspeechpp-libritts960",
    "bench": "stylettszs-librispeech40-mos-s",
    "score": 0.8175,
    "raw_value": 4.27,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'HierSpeech++', column MOS-S (CI) = 4.27 (± 0.12); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). 'LibriTTS-train-960 checkpoint' of github.com/sh-lee-prml/HierSpeechpp (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "HierSpeech++",
    "identity_sources": [
      "https://github.com/sh-lee-prml/HierSpeechpp"
    ]
  },
  {
    "model": "hierspeechpp-libritts960",
    "bench": "stylettszs-librispeech40-hubert-wer",
    "score": 0.0782,
    "raw_value": 7.82,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'HierSpeech++', column WER = 7.82; 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). 'LibriTTS-train-960 checkpoint' of github.com/sh-lee-prml/HierSpeechpp (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "HierSpeech++",
    "identity_sources": [
      "https://github.com/sh-lee-prml/HierSpeechpp"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "stylettszs-librispeech40-mos-n",
    "score": 0.67,
    "raw_value": 3.68,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'XTTSv2', column MOS-N (CI) = 3.68 (± 0.09); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). huggingface.co/coqui/XTTS-v2 (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "stylettszs-librispeech40-mos-s",
    "score": 0.685,
    "raw_value": 3.74,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'XTTSv2', column MOS-S (CI) = 3.74 (± 0.10); 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). huggingface.co/coqui/XTTS-v2 (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "stylettszs-librispeech40-hubert-wer",
    "score": 0.0617,
    "raw_value": 6.17,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2409.10058v1",
    "source_id": "speech-stylettszs-report-bysa",
    "by": "independent",
    "date": "2024-09-16",
    "note": "Table II, row 'XTTSv2', column WER = 6.17; 40 LibriSpeech test-clean sentences with 3-s prompts (VALL-E/NaturalSpeech 2 prompt set), 10 US MTurk raters per set, 1-5 in 0.5 steps, MUSHRA-style hidden side-by-side; WER from HuBERT-large (facebook/hubert-large-ls960-ft). huggingface.co/coqui/XTTS-v2 (Appendix B-A). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "yourtts",
    "bench": "librispeech-test-clean-cross-hubert-wer",
    "score": 0.0792,
    "raw_value": 7.92,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 2, row 'YourTTS', column WER = 7.92; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Table 2 prints '7.92 (7.7*)' and '0.3755 (0.337*)': the unstarred value is the authors' re-run with the official checkpoint github.com/Edresson/YourTTS (starred value copied, not used).",
    "source_model_label": "YourTTS",
    "identity_sources": [
      "https://github.com/Edresson/YourTTS"
    ]
  },
  {
    "model": "yourtts",
    "bench": "librispeech-test-clean-cross-wavlm-sim",
    "score": 0.3755,
    "raw_value": 0.3755,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 2, row 'YourTTS', column SIM-o = 0.3755; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Table 2 prints '7.92 (7.7*)' and '0.3755 (0.337*)': the unstarred value is the authors' re-run with the official checkpoint github.com/Edresson/YourTTS (starred value copied, not used).",
    "source_model_label": "YourTTS",
    "identity_sources": [
      "https://github.com/Edresson/YourTTS"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "librispeech-test-clean-cross-hubert-wer",
    "score": 0.0253,
    "raw_value": 2.53,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 11, row 'StyleTTS 2 ( Li et al., 2024 )', column WER = 2.53; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Appendix A.4: official checkpoint github.com/yl4579/StyleTTS2, objective samples generated from the checkpoint; zero-shot cloning implies the multi-speaker LibriTTS checkpoint (the LJSpeech checkpoint is single-speaker).",
    "source_model_label": "StyleTTS 2",
    "identity_sources": [
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ]
  },
  {
    "model": "styletts-2-libritts",
    "bench": "librispeech-test-clean-cross-wavlm-sim",
    "score": 0.3746,
    "raw_value": 0.3746,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 11, row 'StyleTTS 2 ( Li et al., 2024 )', column SIM-o = 0.3746; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Appendix A.4: official checkpoint github.com/yl4579/StyleTTS2, objective samples generated from the checkpoint; zero-shot cloning implies the multi-speaker LibriTTS checkpoint (the LJSpeech checkpoint is single-speaker).",
    "source_model_label": "StyleTTS 2",
    "identity_sources": [
      "https://huggingface.co/yl4579/StyleTTS2-LibriTTS"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "librispeech-test-clean-cross-hubert-wer",
    "score": 0.0505,
    "raw_value": 5.05,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 11, row 'XTTS-v2 ( Casanova et al., 2024 )', column WER = 5.05; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Appendix A.4: official checkpoint huggingface.co/coqui/XTTS-v2.",
    "source_model_label": "XTTS-v2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "librispeech-test-clean-cross-wavlm-sim",
    "score": 0.4928,
    "raw_value": 0.4928,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 11, row 'XTTS-v2 ( Casanova et al., 2024 )', column SIM-o = 0.4928; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Appendix A.4: official checkpoint huggingface.co/coqui/XTTS-v2.",
    "source_model_label": "XTTS-v2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "yourtts",
    "bench": "voicecraft-zs250-whisper-medium-wer",
    "score": 0.066,
    "raw_value": 6.6,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'YourTTS', column WER = 6.6; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Coqui YourTTS (released multilingual VITS-based zero-shot model). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "YourTTS",
    "identity_sources": [
      "https://github.com/Edresson/YourTTS"
    ]
  },
  {
    "model": "yourtts",
    "bench": "voicecraft-zs250-wavlm-sim",
    "score": 0.41,
    "raw_value": 0.41,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'YourTTS', column SIM = 0.41; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Coqui YourTTS (released multilingual VITS-based zero-shot model). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "YourTTS",
    "identity_sources": [
      "https://github.com/Edresson/YourTTS"
    ]
  },
  {
    "model": "yourtts",
    "bench": "voicecraft-zs80-human-nmos",
    "score": 0.4475,
    "raw_value": 2.79,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'YourTTS', column Naturalness MOS Total = 2.79 ± 0.08; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Coqui YourTTS (released multilingual VITS-based zero-shot model). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "YourTTS",
    "identity_sources": [
      "https://github.com/Edresson/YourTTS"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "voicecraft-zs250-whisper-medium-wer",
    "score": 0.036,
    "raw_value": 3.6,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'XTTS v2', column WER = 3.6; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). huggingface.co/coqui/XTTS-v2 (paper reference). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTS v2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "voicecraft-zs250-wavlm-sim",
    "score": 0.47,
    "raw_value": 0.47,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'XTTS v2', column SIM = 0.47; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). huggingface.co/coqui/XTTS-v2 (paper reference). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTS v2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "voicecraft-zs80-human-nmos",
    "score": 0.74,
    "raw_value": 3.96,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "independent",
    "date": "2024-06-14",
    "note": "Table 6, row 'XTTS v2', column Naturalness MOS Total = 3.96 ± 0.07; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). huggingface.co/coqui/XTTS-v2 (paper reference). CC BY-SA 4.0 (share-alike).",
    "source_model_label": "XTTS v2",
    "identity_sources": [
      "https://huggingface.co/coqui/XTTS-v2"
    ]
  },
  {
    "model": "voicecraft-830m",
    "bench": "voicecraft-zs250-whisper-medium-wer",
    "score": 0.045,
    "raw_value": 4.5,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "lab",
    "date": "2024-06-14",
    "note": "Table 6, row 'VoiceCraft', column WER = 4.5; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Authors' main 830M model (GigaSpeech, 50k steps; same Table 6 values already in v1 of 25 Mar 2024, before the separate 830M_TTSEnhanced release), released as giga830M. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "VoiceCraft",
    "identity_sources": [
      "https://huggingface.co/pyp1/VoiceCraft",
      "https://github.com/jasonppy/VoiceCraft"
    ]
  },
  {
    "model": "voicecraft-830m",
    "bench": "voicecraft-zs250-wavlm-sim",
    "score": 0.55,
    "raw_value": 0.55,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "lab",
    "date": "2024-06-14",
    "note": "Table 6, row 'VoiceCraft', column SIM = 0.55; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Authors' main 830M model (GigaSpeech, 50k steps; same Table 6 values already in v1 of 25 Mar 2024, before the separate 830M_TTSEnhanced release), released as giga830M. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "VoiceCraft",
    "identity_sources": [
      "https://huggingface.co/pyp1/VoiceCraft",
      "https://github.com/jasonppy/VoiceCraft"
    ]
  },
  {
    "model": "voicecraft-830m",
    "bench": "voicecraft-zs80-human-nmos",
    "score": 0.7925,
    "raw_value": 4.17,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2403.16973v3",
    "source_id": "speech-voicecraft-report-bysa",
    "by": "lab",
    "date": "2024-06-14",
    "note": "Table 6, row 'VoiceCraft', column Naturalness MOS Total = 4.17 ± 0.06; authors' zero-shot TTS set of 250 prompt-transcript pairs (half LibriTTS, half GigaSpeech YouTube), ~3-s prompts; WER by Whisper medium.en, SIM by WavLM-TDCNN; MOS from 80 utterances (40 LibriTTS + 40 YouTube), 10 MTurk ratings each. Best-of-5 shortest-sample selection applies to VoiceCraft only (Sec. 4). Authors' main 830M model (GigaSpeech, 50k steps; same Table 6 values already in v1 of 25 Mar 2024, before the separate 830M_TTSEnhanced release), released as giga830M. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "VoiceCraft",
    "identity_sources": [
      "https://huggingface.co/pyp1/VoiceCraft",
      "https://github.com/jasonppy/VoiceCraft"
    ]
  },
  {
    "model": "maskgct",
    "bench": "voxcpm-listening-zh-nmos",
    "score": 0.55,
    "raw_value": 3.2,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'MaskGCT', column ZH N-MOS = 3.20 ± 0.11; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "voxcpm-listening-zh-smos",
    "score": 0.6925,
    "raw_value": 3.77,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'MaskGCT', column ZH S-MOS = 3.77 ± 0.11; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "voxcpm-listening-en-nmos",
    "score": 0.71,
    "raw_value": 3.84,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'MaskGCT', column EN N-MOS = 3.84 ± 0.11; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "voxcpm-listening-en-smos",
    "score": 0.75,
    "raw_value": 4,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'MaskGCT', column EN S-MOS = 4.00 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "voxcpm-listening-zh-nmos",
    "score": 0.595,
    "raw_value": 3.38,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'CosyVoice 2', column ZH N-MOS = 3.38 ± 0.12; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "voxcpm-listening-zh-smos",
    "score": 0.7525,
    "raw_value": 4.01,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'CosyVoice 2', column ZH S-MOS = 4.01 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "voxcpm-listening-en-nmos",
    "score": 0.785,
    "raw_value": 4.14,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'CosyVoice 2', column EN N-MOS = 4.14 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "voxcpm-listening-en-smos",
    "score": 0.7425,
    "raw_value": 3.97,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'CosyVoice 2', column EN S-MOS = 3.97 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "voxcpm-listening-zh-nmos",
    "score": 0.8125,
    "raw_value": 4.25,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'IndexTTS 2', column ZH N-MOS = 4.25 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "IndexTTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "voxcpm-listening-zh-smos",
    "score": 0.7625,
    "raw_value": 4.05,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'IndexTTS 2', column ZH S-MOS = 4.05 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "IndexTTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "voxcpm-listening-en-nmos",
    "score": 0.7575,
    "raw_value": 4.03,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'IndexTTS 2', column EN N-MOS = 4.03 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "IndexTTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "voxcpm-listening-en-smos",
    "score": 0.79,
    "raw_value": 4.16,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Table 5, row 'IndexTTS 2', column EN S-MOS = 4.16 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "IndexTTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "voxcpm-listening-zh-nmos",
    "score": 0.775,
    "raw_value": 4.1,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 5, row 'VoxCPM', column ZH N-MOS = 4.10 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "voxcpm-listening-zh-smos",
    "score": 0.7775,
    "raw_value": 4.11,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 5, row 'VoxCPM', column ZH S-MOS = 4.11 ± 0.10; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "voxcpm-listening-en-nmos",
    "score": 0.7775,
    "raw_value": 4.11,
    "raw_unit": "human naturalness MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 5, row 'VoxCPM', column EN N-MOS = 4.11 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "voxcpm-listening-en-smos",
    "score": 0.795,
    "raw_value": 4.18,
    "raw_unit": "human speaker-similarity MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 5, row 'VoxCPM', column EN S-MOS = 4.18 ± 0.09; listening test, 20 native speakers per language, 5-point scales (Sec. 4.1); test sentences not itemised in the paper. Baselines are released checkpoints (MaskGCT = amphion/MaskGCT, CosyVoice 2 = CosyVoice2-0.5B, IndexTTS 2 = IndexTeam/IndexTTS-2); VoxCPM = the released 0.5B model.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0185,
    "raw_value": 1.85,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 3, row 'VoxCPM' (0.5B), column EN WER = 1.85; own model (openbmb/VoxCPM-0.5B). Seed-TTS-eval official scripts; SIM printed as a percentage (divided by 100 here). Baseline rows of Table 3 are not used (mix of re-runs and copied values).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.729,
    "raw_value": 72.9,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 3, row 'VoxCPM' (0.5B), column EN SIM = 72.9; own model (openbmb/VoxCPM-0.5B). Seed-TTS-eval official scripts; SIM printed as a percentage (divided by 100 here). Baseline rows of Table 3 are not used (mix of re-runs and copied values).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0093,
    "raw_value": 0.93,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 3, row 'VoxCPM' (0.5B), column ZH CER = 0.93; own model (openbmb/VoxCPM-0.5B). Seed-TTS-eval official scripts; SIM printed as a percentage (divided by 100 here). Baseline rows of Table 3 are not used (mix of re-runs and copied values).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.772,
    "raw_value": 77.2,
    "source": "https://arxiv.org/html/2509.24650v1",
    "source_id": "speech-voxcpm-report-ccby",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Table 3, row 'VoxCPM' (0.5B), column ZH SIM = 77.2; own model (openbmb/VoxCPM-0.5B). Seed-TTS-eval official scripts; SIM printed as a percentage (divided by 100 here). Baseline rows of Table 3 are not used (mix of re-runs and copied values).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ],
    "scale_divisor": 100
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.012,
    "raw_value": 1.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2503.01710v1",
    "source_id": "speech-spark-tts-report-ccby",
    "by": "lab",
    "date": "2025-03-03",
    "note": "Table 4, row 'Spark-TTS', column test-zh CER = 1.20; own model (SparkAudio/Spark-TTS-0.5B), Seed-TTS-eval protocol (github.com/BytedanceSpeech/seed-tts-eval). Baseline rows are copied from earlier papers and not used.",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.672,
    "raw_value": 0.672,
    "source": "https://arxiv.org/html/2503.01710v1",
    "source_id": "speech-spark-tts-report-ccby",
    "by": "lab",
    "date": "2025-03-03",
    "note": "Table 4, row 'Spark-TTS', column test-zh SIM = 0.672; own model (SparkAudio/Spark-TTS-0.5B), Seed-TTS-eval protocol (github.com/BytedanceSpeech/seed-tts-eval). Baseline rows are copied from earlier papers and not used.",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0198,
    "raw_value": 1.98,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2503.01710v1",
    "source_id": "speech-spark-tts-report-ccby",
    "by": "lab",
    "date": "2025-03-03",
    "note": "Table 4, row 'Spark-TTS', column test-en WER = 1.98; own model (SparkAudio/Spark-TTS-0.5B), Seed-TTS-eval protocol (github.com/BytedanceSpeech/seed-tts-eval). Baseline rows are copied from earlier papers and not used.",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.584,
    "raw_value": 0.584,
    "source": "https://arxiv.org/html/2503.01710v1",
    "source_id": "speech-spark-tts-report-ccby",
    "by": "lab",
    "date": "2025-03-03",
    "note": "Table 4, row 'Spark-TTS', column test-en SIM = 0.584; own model (SparkAudio/Spark-TTS-0.5B), Seed-TTS-eval protocol (github.com/BytedanceSpeech/seed-tts-eval). Baseline rows are copied from earlier papers and not used.",
    "source_model_label": "Spark-TTS",
    "identity_sources": [
      "https://huggingface.co/SparkAudio/Spark-TTS-0.5B"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0218,
    "raw_value": 2.18,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Index-TTS 2', column WER = 2.18; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'Index-TTS 2' (1.5B, open-weight): IndexTeam/IndexTTS-2.",
    "source_model_label": "Index-TTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.709,
    "raw_value": 0.709,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Index-TTS 2', column SIM = 0.709; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'Index-TTS 2' (1.5B, open-weight): IndexTeam/IndexTTS-2.",
    "source_model_label": "Index-TTS 2",
    "identity_sources": [
      "https://huggingface.co/IndexTeam/IndexTTS-2"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0363,
    "raw_value": 3.63,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Llasa', column WER = 3.63; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'Llasa' with Param. 8B: HKUSTAudio/Llasa-8B (the only 8B Llasa).",
    "source_model_label": "Llasa",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "llasa-8b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.581,
    "raw_value": 0.581,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Llasa', column SIM = 0.581; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'Llasa' with Param. 8B: HKUSTAudio/Llasa-8B (the only 8B Llasa).",
    "source_model_label": "Llasa",
    "identity_sources": [
      "https://huggingface.co/HKUSTAudio/Llasa-8B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0198,
    "raw_value": 1.98,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'VoxCPM', column WER = 1.98; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.73,
    "raw_value": 0.73,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'VoxCPM', column SIM = 0.730; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0261,
    "raw_value": 2.61,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'CosyVoice 2', column WER = 2.61; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.659,
    "raw_value": 0.659,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'CosyVoice 2', column SIM = 0.659; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.025,
    "raw_value": 2.5,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'CosyVoice 3' (the 0.5B open-weight row, not the 1.5B closed-weight row), column WER = 2.50; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.698,
    "raw_value": 0.698,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'CosyVoice 3' (the 0.5B open-weight row, not the 1.5B closed-weight row), column SIM = 0.698; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0146,
    "raw_value": 1.46,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Qwen3-TTS', column WER = 1.46; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.715,
    "raw_value": 0.715,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'Qwen3-TTS', column SIM = 0.715; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0257,
    "raw_value": 2.57,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'MaskGCT', column WER = 2.57; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.713,
    "raw_value": 0.713,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 1, row 'MaskGCT', column SIM = 0.713; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0195,
    "raw_value": 1.95,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 1, row 'Raon-OpenTTS-0.3B', column WER = 1.95; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.687,
    "raw_value": 0.687,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 1, row 'Raon-OpenTTS-0.3B', column SIM = 0.687; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0178,
    "raw_value": 1.78,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 1, row 'Raon-OpenTTS-1B', column WER = 1.78; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.749,
    "raw_value": 0.749,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 1, row 'Raon-OpenTTS-1B', column SIM = 0.749; Seed-TTS-Eval English subset, Whisper-large-v3 WER and WavLM-large speaker-verification SIM 'following the official evaluation protocol'; officially released checkpoints with default/recommended inference settings. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "maskgct",
    "bench": "cv3-eval-en-wer",
    "score": 0.0773,
    "raw_value": 7.73,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'MaskGCT', column CV3-EN WER = 7.73; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.4109,
    "raw_value": 41.09,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'MaskGCT', column CV3-Hard-EN WER = 41.09; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.624,
    "raw_value": 0.624,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'MaskGCT', column CV3-Hard-EN SIM = 0.624; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.62,
    "raw_value": 3.48,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'MaskGCT', column CV3-Hard-EN DNSMOS = 3.48; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-en-wer",
    "score": 0.0627,
    "raw_value": 6.27,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 2', column CV3-EN WER = 6.27; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.1028,
    "raw_value": 10.28,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 2', column CV3-Hard-EN WER = 10.28; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.71,
    "raw_value": 0.71,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 2', column CV3-Hard-EN SIM = 0.710; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.7375,
    "raw_value": 3.95,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 2', column CV3-Hard-EN DNSMOS = 3.95; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-en-wer",
    "score": 0.0496,
    "raw_value": 4.96,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 3', column CV3-EN WER = 4.96; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.1077,
    "raw_value": 10.77,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 3', column CV3-Hard-EN WER = 10.77; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.74,
    "raw_value": 0.74,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 3', column CV3-Hard-EN SIM = 0.740; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.745,
    "raw_value": 3.98,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'CosyVoice 3', column CV3-Hard-EN DNSMOS = 3.98; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-en-wer",
    "score": 0.0524,
    "raw_value": 5.24,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'VoxCPM', column CV3-EN WER = 5.24; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0644,
    "raw_value": 6.44,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'VoxCPM', column CV3-Hard-EN WER = 6.44; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.67,
    "raw_value": 0.67,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'VoxCPM', column CV3-Hard-EN SIM = 0.670; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.695,
    "raw_value": 3.78,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'VoxCPM', column CV3-Hard-EN DNSMOS = 3.78; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-en-wer",
    "score": 0.0452,
    "raw_value": 4.52,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'Qwen3-TTS', column CV3-EN WER = 4.52; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0789,
    "raw_value": 7.89,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'Qwen3-TTS', column CV3-Hard-EN WER = 7.89; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.666,
    "raw_value": 0.666,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'Qwen3-TTS', column CV3-Hard-EN SIM = 0.666; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.7175,
    "raw_value": 3.87,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 5, row 'Qwen3-TTS', column CV3-Hard-EN DNSMOS = 3.87; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "cv3-eval-en-wer",
    "score": 0.0462,
    "raw_value": 4.62,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-0.3B', column CV3-EN WER = 4.62; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0731,
    "raw_value": 7.31,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-0.3B', column CV3-Hard-EN WER = 7.31; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.73,
    "raw_value": 0.73,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-0.3B', column CV3-Hard-EN SIM = 0.730; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.6925,
    "raw_value": 3.77,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-0.3B', column CV3-Hard-EN DNSMOS = 3.77; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "cv3-eval-en-wer",
    "score": 0.0392,
    "raw_value": 3.92,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-1B', column CV3-EN WER = 3.92; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0615,
    "raw_value": 6.15,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-1B', column CV3-Hard-EN WER = 6.15; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.775,
    "raw_value": 0.775,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-1B', column CV3-Hard-EN SIM = 0.775; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.7125,
    "raw_value": 3.85,
    "raw_unit": "DNSMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 5, row 'Raon-OpenTTS-1B', column CV3-Hard-EN DNSMOS = 3.85; CV3-Eval official protocol: EN subset WER; Hard-EN WER, ERes2Net speaker-verification SIM and DNSMOS. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "maskgct",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.0861,
    "raw_value": 8.61,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'MaskGCT', column Overall WER = 8.61; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.635,
    "raw_value": 0.635,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'MaskGCT', column Overall SIM = 0.635; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "maskgct",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.645,
    "raw_value": 3.58,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 8, row 'MaskGCT', column Overall = 3.58; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'MaskGCT' (0.6B): amphion/MaskGCT.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://huggingface.co/amphion/MaskGCT"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.1102,
    "raw_value": 11.02,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'CosyVoice 2', column Overall WER = 11.02; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.603,
    "raw_value": 0.603,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'CosyVoice 2', column Overall SIM = 0.603; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.6325,
    "raw_value": 3.53,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 8, row 'CosyVoice 2', column Overall = 3.53; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 2' (0.5B): FunAudioLLM/CosyVoice2-0.5B.",
    "source_model_label": "CosyVoice 2",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.0443,
    "raw_value": 4.43,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'CosyVoice 3', column Overall WER = 4.43; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.647,
    "raw_value": 0.647,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'CosyVoice 3', column Overall SIM = 0.647; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.645,
    "raw_value": 3.58,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 8, row 'CosyVoice 3', column Overall = 3.58; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'CosyVoice 3' 0.5B open-weight ('officially released 0.5B checkpoint'): FunAudioLLM/Fun-CosyVoice3-0.5B-2512, the only released CosyVoice 3 weights; 'default ... inference settings' imply the default llm.pt, not the optional llm.rl.pt.",
    "source_model_label": "CosyVoice 3",
    "identity_sources": [
      "https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.0948,
    "raw_value": 9.48,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'VoxCPM', column Overall WER = 9.48; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.642,
    "raw_value": 0.642,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'VoxCPM', column Overall SIM = 0.642; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.635,
    "raw_value": 3.54,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 8, row 'VoxCPM', column Overall = 3.54; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Table 1 row 'VoxCPM' with Param. 0.5B and 1.8M h: openbmb/VoxCPM-0.5B (VoxCPM1.5 is 0.8B; VoxCPM2 is a separate model).",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://huggingface.co/openbmb/VoxCPM-0.5B"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.1759,
    "raw_value": 17.59,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'Qwen3-TTS', column Overall WER = 17.59; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.626,
    "raw_value": 0.626,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 6, row 'Qwen3-TTS', column Overall SIM = 0.626; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.6475,
    "raw_value": 3.59,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "independent",
    "date": "2026-06-15",
    "note": "Table 8, row 'Qwen3-TTS', column Overall = 3.59; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: row 'Qwen3-TTS' (1.7B, open-weight) used for zero-shot cloning: Qwen/Qwen3-TTS-12Hz-1.7B-Base, the only released 1.7B cloning checkpoint (25 Hz variants unreleased; CustomVoice/VoiceDesign do not clone).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.0293,
    "raw_value": 2.93,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 6, row 'Raon-OpenTTS-0.3B', column Overall WER = 2.93; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.623,
    "raw_value": 0.623,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 6, row 'Raon-OpenTTS-0.3B', column Overall SIM = 0.623; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-0-3b",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.65,
    "raw_value": 3.6,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 8, row 'Raon-OpenTTS-0.3B', column Overall = 3.60; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-0.3B (KRAFTON/Raon-OpenTTS-0.3B; F5-TTS DiT, 336M, Raon-OpenTTS-Core 510K h, 32 NFE, HiFi-GAN 16 kHz).",
    "source_model_label": "Raon-OpenTTS-0.3B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-0.3B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "raon-opentts-eval-overall-wer",
    "score": 0.0281,
    "raw_value": 2.81,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 6, row 'Raon-OpenTTS-1B', column Overall WER = 2.81; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "raon-opentts-eval-overall-wavlm-sim",
    "score": 0.695,
    "raw_value": 0.695,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 6, row 'Raon-OpenTTS-1B', column Overall SIM = 0.695; Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "raon-opentts-1b",
    "bench": "raon-opentts-eval-human-smos",
    "score": 0.675,
    "raw_value": 3.7,
    "raw_unit": "human SMOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2605.20830v2",
    "source_id": "speech-raon-opentts-report-ccby",
    "by": "lab",
    "date": "2026-06-15",
    "note": "Table 8, row 'Raon-OpenTTS-1B', column Overall = 3.70; SMOS on 30 items per acoustic condition, 6 US MTurk annotators per item, 5-point scale (Appendix B); Raon-OpenTTS-Eval (6,000 prompt-text pairs from 12 English corpora in Clean/Noisy/Wild/Expressive conditions; Whisper-normalised WER; SIM = WavLM speaker-embedding cosine between prompt and output). Overall = all samples pooled. Identity: Authors' released Raon-OpenTTS-1B (KRAFTON/Raon-OpenTTS-1B; 1,048M, Raon-OpenTTS-Core 510K h, 32 NFE).",
    "source_model_label": "Raon-OpenTTS-1B",
    "identity_sources": [
      "https://huggingface.co/KRAFTON/Raon-OpenTTS-1B",
      "https://github.com/krafton-ai/RAON-OpenTTS/blob/0dd28405c9348e9505c2a1c92a250cb3beffd950/README.md"
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.58,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK', LibriTTS block, row 'CosyVoice': SIM 0.58 ± 0.01, MOS 4.37, WER 0.05 ± 0.00. This column: SIM = 0.58. Replaces the v2 value 0.60 (row 'CozyVoice'); the other ten v2 systems print unchanged LibriTTS SIM/WER in v3. Identity: paper text names CosyVoice 2 (Du et al., 2024); repo config loads pretrained_models/CosyVoice2-0.5B (FunAudioLLM/CosyVoice2-0.5B). Default inference settings; weight revisions not pinned. Re-read 2026-10-11.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.41,
    "raw_value": 0.41,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'SparkAudio': SIM 0.41 ± 0.01, MOS 4.06, WER 0.33 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.41. Identity: Paper Table 3 'SparkTTS 0.5B' (Wang et al., 2025c); v1 table label 'SparkTTS'; repo config loads pretrained_models/Spark-TTS-0.5B (SparkAudio/Spark-TTS-0.5B). Default inference settings; weight revisions not pinned.",
    "source_model_label": "SparkAudio",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/b742742fed101df02a25dbda239b69d02e685c96/configs/ots_vc/protection/denoiser_dns64_on_spec_libritts/sparktts_ots.yaml"
    ]
  },
  {
    "model": "higgs-audio-v2",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.56,
    "raw_value": 0.56,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'Higgs': SIM 0.56 ± 0.00, MOS 4.30, WER 0.25 ± 0.03 (1000-sample bootstrap). This column: SIM = 0.56. Identity: Paper links boson-ai/higgs-audio and the Higgs Audio v2 release blog; repo config model_path bosonai/higgs-audio-v2-generation-3B-base (the released v2 generation checkpoint). Default inference settings; weight revisions not pinned.",
    "source_model_label": "Higgs",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/b742742fed101df02a25dbda239b69d02e685c96/configs/ots_vc/protection/denoiser_dns64_on_spec_libritts/higgs_audio_ots.yaml",
      "https://github.com/boson-ai/higgs-audio"
    ]
  },
  {
    "model": "vibevoice-1-5b",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.48,
    "raw_value": 0.48,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'VibeVoice': SIM 0.48 ± 0.00, MOS 3.83, WER 0.23 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.48. Identity: Repo config model_path vibevoice/VibeVoice-1.5B. Default inference settings; weight revisions not pinned.",
    "source_model_label": "VibeVoice",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/b742742fed101df02a25dbda239b69d02e685c96/configs/ots_vc/protection/denoiser_dns64_on_spec_libritts/vibevoice_ots.yaml"
    ]
  },
  {
    "model": "mgm-omni-tts-2b-0927",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.54,
    "raw_value": 0.54,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'MGM-Omni': SIM 0.54 ± 0.00, MOS 4.28, WER 0.09 ± 0.00 (1000-sample bootstrap). This column: SIM = 0.54. Identity: Paper cites MGM-Omni (Wang et al., 2025a); repo config checkpoint_path wcy1122/MGM-Omni-TTS-2B-0927. Default inference settings; weight revisions not pinned.",
    "source_model_label": "MGM-Omni",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/b742742fed101df02a25dbda239b69d02e685c96/configs/ots_vc/protection/denoiser_dns64_on_spec_libritts/mgm_omni_ots.yaml",
      "https://huggingface.co/wcy1122/MGM-Omni-TTS-2B-0927"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.61,
    "raw_value": 0.61,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'Qwen-TTS': SIM 0.61 ± 0.00, MOS 4.39, WER 0.05 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.61. Identity: Paper text names Qwen3-TTS (Hu et al., 2026); repo LibriTTS config checkpoint_path Qwen/Qwen3-TTS-12Hz-1.7B-Base (reference-audio cloning). Default inference settings; weight revisions not pinned.",
    "source_model_label": "Qwen-TTS",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/qwen3_tts_ots.yaml"
    ]
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.56,
    "raw_value": 0.56,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'F5-TTS': SIM 0.56 ± 0.00, MOS 3.99, WER 0.12 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.56. Identity: Repo LibriTTS config adversary.model F5TTS_v1_Base (32 NFE, CFG 2.0, sway -1, EMA); added in v2 of the paper. Default inference settings; weight revisions not pinned.",
    "source_model_label": "F5-TTS",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/f5_tts_ots.yaml"
    ]
  },
  {
    "model": "maskgct",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.57,
    "raw_value": 0.57,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'MaskGCT': SIM 0.57 ± 0.00, MOS 3.93, WER 0.09 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.57. Identity: Repo LibriTTS config repo_id amphion/MaskGCT (25 T2S steps, cfg 2.5). Default inference settings; weight revisions not pinned.",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/maskgct_ots.yaml"
    ]
  },
  {
    "model": "openvoice-v2",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.24,
    "raw_value": 0.24,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'OpenVoice': SIM 0.24 ± 0.00, MOS 4.30, WER 0.07 ± 0.00 (1000-sample bootstrap). This column: SIM = 0.24. Identity: Repo LibriTTS config uses OpenVoice checkpoints_v2 converter + base speakers (MeloTTS via HF), tau 0.3. Default inference settings; weight revisions not pinned.",
    "source_model_label": "OpenVoice",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/openvoice_ots.yaml",
      "https://huggingface.co/myshell-ai/OpenVoiceV2"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.45,
    "raw_value": 0.45,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'XTTS': SIM 0.45 ± 0.00, MOS 3.81, WER 0.07 ± 0.01 (1000-sample bootstrap). This column: SIM = 0.45. Identity: Repo LibriTTS config checkpoint coqui/XTTS-v2 (strict_checkpoint true). Default inference settings; weight revisions not pinned.",
    "source_model_label": "XTTS",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/xtts_ots.yaml"
    ]
  },
  {
    "model": "zipvoice",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.58,
    "raw_value": 0.58,
    "scale_divisor": 1,
    "source": "https://arxiv.org/html/2602.00443v2",
    "source_id": "speech-rvcbench-ccby",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1 'Performance on LibriTTS and VCTK'), LibriTTS block, row 'ZipVoice': SIM 0.58 ± 0.00, MOS 4.13, WER 0.05 ± 0.00 (1000-sample bootstrap). This column: SIM = 0.58. Identity: Repo LibriTTS config model_name zipvoice (k2-fsa/ZipVoice, not the distilled zipvoice_distill), checkpoint model.pt. Default inference settings; weight revisions not pinned.",
    "source_model_label": "ZipVoice",
    "identity_sources": [
      "https://github.com/Nanboy-Ronan/RVCBench/blob/171c7cb3397cb4acabd89b4dfc0751ba52ffcf75/configs/ots_vc/clean/libritts/zipvoice_ots.yaml",
      "https://huggingface.co/k2-fsa/ZipVoice"
    ]
  },
  {
    "model": "wayu-paxa-tts-edge",
    "bench": "wayu-thai-cer500-typhoon-whisper",
    "score": 0.037,
    "raw_value": 3.7,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "lab",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'Wayu-Paxa-TTS-Edge': Keyword 68.2%, CER Thai 3.7%, CER Eng. 1.1%. Identity: Authors' own released 82M fixed-voice student (Kokoro/StyleTTS2 backbone), HF wayu-ai/wayu-paxa-tts-edge. This column: CER Thai = 3.7%.",
    "source_model_label": "Wayu-Paxa-TTS-Edge",
    "identity_sources": [
      "https://huggingface.co/wayu-ai/wayu-paxa-tts-edge"
    ]
  },
  {
    "model": "wayu-paxa-tts-edge",
    "bench": "wayu-thai-challenge-keyword-acc",
    "score": 0.682,
    "raw_value": 68.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "lab",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'Wayu-Paxa-TTS-Edge': Keyword 68.2%, CER Thai 3.7%, CER Eng. 1.1%. Identity: Authors' own released 82M fixed-voice student (Kokoro/StyleTTS2 backbone), HF wayu-ai/wayu-paxa-tts-edge. This column: Keyword = 68.2%.",
    "source_model_label": "Wayu-Paxa-TTS-Edge",
    "identity_sources": [
      "https://huggingface.co/wayu-ai/wayu-paxa-tts-edge"
    ]
  },
  {
    "model": "omnivoice",
    "bench": "wayu-thai-cer500-typhoon-whisper",
    "score": 0.046,
    "raw_value": 4.6,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'OmniVoice teacher': Keyword 72.8%, CER Thai 4.6%, CER Eng. 0.9%. Identity: k2-fsa/OmniVoice (600M) zero-shot voice-cloning model run by the authors as their teacher from a 15 s voice reference; paper footnote links huggingface.co/k2-fsa/OmniVoice. This column: CER Thai = 4.6%.",
    "source_model_label": "OmniVoice teacher",
    "identity_sources": [
      "https://huggingface.co/k2-fsa/OmniVoice"
    ]
  },
  {
    "model": "omnivoice",
    "bench": "wayu-thai-challenge-keyword-acc",
    "score": 0.728,
    "raw_value": 72.8,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'OmniVoice teacher': Keyword 72.8%, CER Thai 4.6%, CER Eng. 0.9%. Identity: k2-fsa/OmniVoice (600M) zero-shot voice-cloning model run by the authors as their teacher from a 15 s voice reference; paper footnote links huggingface.co/k2-fsa/OmniVoice. This column: Keyword = 72.8%.",
    "source_model_label": "OmniVoice teacher",
    "identity_sources": [
      "https://huggingface.co/k2-fsa/OmniVoice"
    ]
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "wayu-thai-cer500-typhoon-whisper",
    "score": 0.033,
    "raw_value": 3.3,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'Gemini 3.1 Flash TTS': Keyword 79.8%, CER Thai 3.3%, CER Eng. 0.8%. Identity: Paper footnote links ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview; prebuilt voice Kore (speaker similarity uses Kore references). This column: CER Thai = 3.3%.",
    "source_model_label": "Gemini 3.1 Flash TTS",
    "identity_sources": [
      "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview"
    ]
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "wayu-thai-challenge-keyword-acc",
    "score": 0.798,
    "raw_value": 79.8,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2609.03502v1",
    "source_id": "speech-wayu-thai-tts-ccby",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Table 8 ('Final comparison'), row 'Gemini 3.1 Flash TTS': Keyword 79.8%, CER Thai 3.3%, CER Eng. 0.8%. Identity: Paper footnote links ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview; prebuilt voice Kore (speaker similarity uses Kore references). This column: Keyword = 79.8%.",
    "source_model_label": "Gemini 3.1 Flash TTS",
    "identity_sources": [
      "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview"
    ]
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.2862,
    "raw_value": 28.62,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Tortoise-TTS (links https://github.com/neonbjb/tortoise-tts); voice random; no strong-prompting star. Stable Gemini judge six-category overall WER column (Whisper-large-v3). Official neonbjb/tortoise-tts repo (only released weights: jbetker/tortoise-tts-v2); paper §4 lists Tortoise-TTS among evaluated open-source models; audio clipped ~27 s (paper appendix). Weight revision not pinned.",
    "source_model_label": "Tortoise-TTS",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/neonbjb/tortoise-tts"
    ],
    "scale_divisor": 100,
    "voice": "random",
    "strong_prompting": false
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.1636,
    "raw_value": 16.36,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Tortoise-TTS; voice random; no strong-prompting star. Stable Gemini judge six-category overall win-rate vs gpt-4o-mini-tts Alloy baseline. Official neonbjb/tortoise-tts repo (only released weights: jbetker/tortoise-tts-v2); paper §4 lists Tortoise-TTS among evaluated open-source models; audio clipped ~27 s (paper appendix).",
    "source_model_label": "Tortoise-TTS",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/neonbjb/tortoise-tts"
    ],
    "scale_divisor": 100,
    "voice": "random",
    "strong_prompting": false
  },
  {
    "model": "bark",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.2071,
    "raw_value": 20.71,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Suno Bark (links https://github.com/suno-ai/bark); voice v2/en_speaker_6; no strong-prompting star. Stable Gemini judge six-category overall WER column (Whisper-large-v3). Official suno-ai/bark repo default (full-size) checkpoints, preset v2/en_speaker_6; long texts split per official recipe (paper appendix). Weight revision not pinned.",
    "source_model_label": "Suno Bark",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/suno-ai/bark"
    ],
    "scale_divisor": 100,
    "voice": "v2/en_speaker_6",
    "strong_prompting": false
  },
  {
    "model": "bark",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.0902,
    "raw_value": 9.02,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row Suno Bark; voice v2/en_speaker_6; no strong-prompting star. Stable Gemini judge six-category overall win-rate vs gpt-4o-mini-tts Alloy baseline. Official suno-ai/bark repo default (full-size) checkpoints, preset v2/en_speaker_6; long texts split per official recipe (paper appendix).",
    "source_model_label": "Suno Bark",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/suno-ai/bark"
    ],
    "scale_divisor": 100,
    "voice": "v2/en_speaker_6",
    "strong_prompting": false
  },
  {
    "model": "vits-vctk",
    "bench": "emergent-tts-6cat-stable-wer",
    "score": 0.2745,
    "raw_value": 27.45,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row VITS-VCTK (links https://github.com/jaywalnut310/vits); voice default; no strong-prompting star. Stable Gemini judge six-category overall WER column (Whisper-large-v3). jaywalnut310/vits official VCTK multi-speaker checkpoint (pretrained_vctk.pth, the only released VCTK model); row added to README 2025-07-30 (commit 9d581ad). Weight revision not pinned.",
    "source_model_label": "VITS-VCTK",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/jaywalnut310/vits"
    ],
    "scale_divisor": 100,
    "voice": "default",
    "strong_prompting": false
  },
  {
    "model": "vits-vctk",
    "bench": "emergent-tts-6cat-stable-preference",
    "score": 0.0847,
    "raw_value": 8.47,
    "source": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
    "source_id": "speech-emergent-apache",
    "by": "independent",
    "date": "2025-08-10",
    "note": "README row VITS-VCTK; voice default; no strong-prompting star. Stable Gemini judge six-category overall win-rate vs gpt-4o-mini-tts Alloy baseline. jaywalnut310/vits official VCTK multi-speaker checkpoint (pretrained_vctk.pth, the only released VCTK model); row added to README 2025-07-30 (commit 9d581ad).",
    "source_model_label": "VITS-VCTK",
    "identity_sources": [
      "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md",
      "https://github.com/jaywalnut310/vits"
    ],
    "scale_divisor": 100,
    "voice": "default",
    "strong_prompting": false
  },
  {
    "model": "bark",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.6525,
    "raw_value": 3.61,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'Suno Bark'; voice v2/en_speaker_6; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 20.71. Raw MOS 3.61; normalized (MOS-1)/4.",
    "source_model_label": "Suno Bark",
    "identity_sources": [
      "https://arxiv.org/html/2505.23009v1"
    ],
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "scale_offset": 1,
    "scale_divisor": 4,
    "voice": "v2/en_speaker_6",
    "strong_prompting": false
  },
  {
    "model": "tortoise-tts-v2",
    "bench": "emergent-tts-v1-paper-wvmos",
    "score": 0.5075,
    "raw_value": 3.03,
    "source": "https://arxiv.org/html/2505.23009v1",
    "source_id": "speech-emergent-v1-paper-ccby",
    "by": "independent",
    "date": "2025-05-29",
    "note": "Paper Table 1 row 'Tortoise-TTS'; voice random; no strong prompting. MOS column = WV-MOS automatic predictor averaged over the 1,645 generations. Overall WER in same row 28.62. Raw MOS 3.03; normalized (MOS-1)/4.",
    "source_model_label": "Tortoise-TTS",
    "identity_sources": [
      "https://arxiv.org/html/2505.23009v1"
    ],
    "raw_unit": "predicted MOS (1-5 scale)",
    "scale_transform": "(raw-1)/4",
    "scale_offset": 1,
    "scale_divisor": 4,
    "voice": "random",
    "strong_prompting": false
  },
  {
    "model": "outetts-0-2-500m",
    "bench": "cloneval-ls-test-clean-wavlm-sim",
    "score": 0.8835977458,
    "raw_value": 0.8835977457577491,
    "source": "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
    "source_id": "speech-cloneval-space-apache",
    "by": "independent",
    "date": "2025-04-09",
    "note": "results.csv row emotion=All, model=OuteAI/OuteTTS-0.2-500M, dataset=LS Clean, column wavlm (mean cosine similarity of microsoft/wavlm-base-plus-sv embeddings between reference and cloned sample) = 0.8835977457577491. ClonEval paper (arXiv:2504.20581v3, identity only) Table 1 prints 0.8836. Generated by the ClonEval authors (AMU); checkpoint revision not pinned.",
    "source_model_label": "OuteAI/OuteTTS-0.2-500M",
    "identity_sources": [
      "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
      "https://arxiv.org/abs/2504.20581v3",
      "https://huggingface.co/OuteAI/OuteTTS-0.2-500M"
    ],
    "date_basis": "commit d6e4dd3 \"Fix WavLM results\" (2025-04-09) last changed the values"
  },
  {
    "model": "vall-e-x-plachtaa",
    "bench": "cloneval-ls-test-clean-wavlm-sim",
    "score": 0.9009997314,
    "raw_value": 0.9009997314169208,
    "source": "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
    "source_id": "speech-cloneval-space-apache",
    "by": "independent",
    "date": "2025-04-09",
    "note": "results.csv row emotion=All, model=Plachtaa/VALL-E-X, dataset=LS Clean, column wavlm (mean cosine similarity of microsoft/wavlm-base-plus-sv embeddings between reference and cloned sample) = 0.9009997314169208. ClonEval paper (arXiv:2504.20581v3, identity only) Table 1 prints 0.9010. Generated by the ClonEval authors (AMU); checkpoint revision not pinned.",
    "source_model_label": "Plachtaa/VALL-E-X",
    "identity_sources": [
      "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
      "https://arxiv.org/abs/2504.20581v3",
      "https://github.com/Plachtaa/VALL-E-X"
    ],
    "date_basis": "commit d6e4dd3 \"Fix WavLM results\" (2025-04-09) last changed the values"
  },
  {
    "model": "xtts-v2",
    "bench": "cloneval-ls-test-clean-wavlm-sim",
    "score": 0.8881450431,
    "raw_value": 0.8881450431037495,
    "source": "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
    "source_id": "speech-cloneval-space-apache",
    "by": "independent",
    "date": "2025-04-09",
    "note": "results.csv row emotion=All, model=coqui/XTTS-v2, dataset=LS Clean, column wavlm (mean cosine similarity of microsoft/wavlm-base-plus-sv embeddings between reference and cloned sample) = 0.8881450431037495. ClonEval paper (arXiv:2504.20581v3, identity only) Table 1 prints 0.8881. Generated by the ClonEval authors (AMU); checkpoint revision not pinned.",
    "source_model_label": "coqui/XTTS-v2",
    "identity_sources": [
      "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv",
      "https://arxiv.org/abs/2504.20581v3",
      "https://huggingface.co/coqui/XTTS-v2"
    ],
    "date_basis": "commit d6e4dd3 \"Fix WavLM results\" (2025-04-09) last changed the values"
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1147.5,
    "raw_value": 1147.5,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id gemini_flash_tts 'Gemini 3.1 Flash TTS': elo 1147.5 ±26.9, 517 battles (W235/L125/T157), active (not retired). Identity: served as google/gemini-3.1-flash-tts-preview (arena model_url); matches catalogue exact id gemini-3.1-flash-tts-preview.",
    "source_model_label": "Gemini 3.1 Flash TTS",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://openrouter.ai/google/gemini-3.1-flash-tts-preview"
    ]
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1072.8,
    "raw_value": 1072.8,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id higgs_tts_3 'Higgs TTS 3': elo 1072.8 ±24.6, 551 battles (W147/L156/T248), active (not retired). Identity: bosonai/higgs-tts-3-4b; HF redirects bosonai/higgs-audio-v3-tts-4b to the same repo (sha 239f63f), catalogue note records the alias.",
    "source_model_label": "Higgs TTS 3",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/bosonai/higgs-tts-3-4b"
    ]
  },
  {
    "model": "voxcpm2",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1064,
    "raw_value": 1064,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id voxcpm2 'VoxCPM 2': elo 1064.0 ±19.0, 1040 battles (W325/L255/T460), active (not retired). Identity: openbmb/VoxCPM2.",
    "source_model_label": "VoxCPM 2",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/openbmb/VoxCPM2"
    ]
  },
  {
    "model": "xai-grok-tts",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1063.4,
    "raw_value": 1063.4,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id grok_voice_tts 'Grok Voice TTS 1.0': elo 1063.4 ±26.6, 561 battles (W165/L186/T210), active (not retired). Identity: \"Grok Voice TTS 1.0\"; xAI first-party model list (docs.x.ai/docs/models) has exactly one audio TTS model: grok-tts version 1.0.",
    "source_model_label": "Grok Voice TTS 1.0",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://openrouter.ai/x-ai/grok-voice-tts-1.0"
    ]
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1057.4,
    "raw_value": 1057.4,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id fish_speech_s2 'Fish Speech S2 Pro': elo 1057.4 ±19.5, 997 battles (W315/L252/T430), active (not retired). Identity: fishaudio/s2-pro open weights.",
    "source_model_label": "Fish Speech S2 Pro",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/fishaudio/s2-pro"
    ]
  },
  {
    "model": "supertonic-3",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1051.4,
    "raw_value": 1051.4,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id supertonic_3 'Supertonic 3': elo 1051.4 ±19.3, 831 battles (W221/L235/T375), active (not retired). Identity: Supertone/supertonic-3.",
    "source_model_label": "Supertonic 3",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/Supertone/supertonic-3"
    ]
  },
  {
    "model": "omnivoice",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1049.4,
    "raw_value": 1049.4,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id omnivoice 'OmniVoice': elo 1049.4 ±19.3, 1020 battles (W309/L290/T421), active (not retired). Identity: k2-fsa/OmniVoice.",
    "source_model_label": "OmniVoice",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/k2-fsa/OmniVoice"
    ]
  },
  {
    "model": "chatterbox-multilingual-v3",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1043.3,
    "raw_value": 1043.3,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id chatterbox_multilingual_v3 'Chatterbox Multilingual V3': elo 1043.3 ±23.0, 554 battles (W119/L173/T262), active (not retired). Identity: ResembleAI/chatterbox, Chatterbox Multilingual V3 (500M) release; distinct from original English Chatterbox and the retired earlier \"Multilingual Chatterbox\" arena entry.",
    "source_model_label": "Chatterbox Multilingual V3",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/ResembleAI/chatterbox"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1014.6,
    "raw_value": 1014.6,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "source_id": "speech-arabic-tts-arena-apache",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id xtts_v2 'XTTS v2': elo 1014.6 ±17.9, 1068 battles (W265/L326/T477), active (not retired). Identity: tts-hub/XTTS-v2 mirror of coqui/XTTS-v2.",
    "source_model_label": "XTTS v2",
    "identity_sources": [
      "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
      "https://huggingface.co/tts-hub/XTTS-v2"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "balacoon-vctk-zero-tts-whisper-cer",
    "score": 0.0095638592,
    "raw_value": 0.009563859185389883,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2026-01-24",
    "note": "zero-tts/qwen3_tts/vctk/metrics.yaml (model_name Qwen3-TTS). Identity: Qwen/Qwen3-TTS-12Hz-1.7B-Base (notebooks/qwen3_tts.ipynb from_pretrained). whisperv3_cer = 0.009563859185389883 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/qwen3_tts.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "balacoon-vctk-zero-tts-redimnet-sim",
    "score": 0.7357676678,
    "raw_value": 0.7357676677916454,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2026-01-24",
    "note": "zero-tts/qwen3_tts/vctk/metrics.yaml (model_name Qwen3-TTS). Identity: Qwen/Qwen3-TTS-12Hz-1.7B-Base (notebooks/qwen3_tts.ipynb from_pretrained). redimnet_secs = 0.7357676677916454 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/qwen3_tts.ipynb"
    ]
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "balacoon-daps-celeb-zero-tts-whisper-cer",
    "score": 0.0335527865,
    "raw_value": 0.03355278650926106,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2026-01-24",
    "note": "zero-tts/qwen3_tts/daps_celeb/metrics.yaml (model_name Qwen3-TTS). Identity: Qwen/Qwen3-TTS-12Hz-1.7B-Base (notebooks/qwen3_tts.ipynb from_pretrained). whisperv3_cer = 0.03355278650926106 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/qwen3_tts.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "balacoon-daps-celeb-zero-tts-redimnet-sim",
    "score": 0.7209245161,
    "raw_value": 0.7209245161183099,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2026-01-24",
    "note": "zero-tts/qwen3_tts/daps_celeb/metrics.yaml (model_name Qwen3-TTS). Identity: Qwen/Qwen3-TTS-12Hz-1.7B-Base (notebooks/qwen3_tts.ipynb from_pretrained). redimnet_secs = 0.7209245161183099 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Qwen3-TTS",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/qwen3_tts/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/qwen3_tts.ipynb"
    ]
  },
  {
    "model": "sesame-csm-1b",
    "bench": "balacoon-vctk-zero-tts-whisper-cer",
    "score": 0.0193503054,
    "raw_value": 0.019350305388897898,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-03-23",
    "note": "zero-tts/sesame/vctk/metrics.yaml (model_name Sesame-csm-1b). Identity: sesame/csm-1b (metrics.yaml link). whisperv3_cer = 0.019350305388897898 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Sesame-csm-1b",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/vctk/metrics.yaml",
      "https://huggingface.co/sesame/csm-1b"
    ],
    "scale_divisor": 1
  },
  {
    "model": "sesame-csm-1b",
    "bench": "balacoon-vctk-zero-tts-redimnet-sim",
    "score": 0.651523351,
    "raw_value": 0.6515233510397375,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-03-23",
    "note": "zero-tts/sesame/vctk/metrics.yaml (model_name Sesame-csm-1b). Identity: sesame/csm-1b (metrics.yaml link). redimnet_secs = 0.6515233510397375 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Sesame-csm-1b",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/vctk/metrics.yaml",
      "https://huggingface.co/sesame/csm-1b"
    ]
  },
  {
    "model": "sesame-csm-1b",
    "bench": "balacoon-daps-celeb-zero-tts-whisper-cer",
    "score": 0.0854265698,
    "raw_value": 0.08542656979938372,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-03-23",
    "note": "zero-tts/sesame/daps_celeb/metrics.yaml (model_name Sesame-csm-1b). Identity: sesame/csm-1b (metrics.yaml link). whisperv3_cer = 0.08542656979938372 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Sesame-csm-1b",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/daps_celeb/metrics.yaml",
      "https://huggingface.co/sesame/csm-1b"
    ],
    "scale_divisor": 1
  },
  {
    "model": "sesame-csm-1b",
    "bench": "balacoon-daps-celeb-zero-tts-redimnet-sim",
    "score": 0.67317507,
    "raw_value": 0.6731750699684048,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-03-23",
    "note": "zero-tts/sesame/daps_celeb/metrics.yaml (model_name Sesame-csm-1b). Identity: sesame/csm-1b (metrics.yaml link). redimnet_secs = 0.6731750699684048 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Sesame-csm-1b",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/sesame/daps_celeb/metrics.yaml",
      "https://huggingface.co/sesame/csm-1b"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "balacoon-vctk-zero-tts-whisper-cer",
    "score": 0.0083238744,
    "raw_value": 0.00832387438516837,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/xtts/vctk/metrics.yaml (model_name XTTSv2). Identity: TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\") in notebooks/xtts.ipynb. whisperv3_cer = 0.00832387438516837 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/xtts.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "xtts-v2",
    "bench": "balacoon-vctk-zero-tts-redimnet-sim",
    "score": 0.5769889716,
    "raw_value": 0.5769889716058969,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/xtts/vctk/metrics.yaml (model_name XTTSv2). Identity: TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\") in notebooks/xtts.ipynb. redimnet_secs = 0.5769889716058969 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/xtts.ipynb"
    ]
  },
  {
    "model": "xtts-v2",
    "bench": "balacoon-daps-celeb-zero-tts-whisper-cer",
    "score": 0.0276204233,
    "raw_value": 0.027620423330165275,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/xtts/daps_celeb/metrics.yaml (model_name XTTSv2). Identity: TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\") in notebooks/xtts.ipynb. whisperv3_cer = 0.027620423330165275 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/xtts.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "xtts-v2",
    "bench": "balacoon-daps-celeb-zero-tts-redimnet-sim",
    "score": 0.529142188,
    "raw_value": 0.5291421880451636,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/xtts/daps_celeb/metrics.yaml (model_name XTTSv2). Identity: TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\") in notebooks/xtts.ipynb. redimnet_secs = 0.5291421880451636 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "XTTSv2",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/xtts/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/xtts.ipynb"
    ]
  },
  {
    "model": "zonos-v0-1",
    "bench": "balacoon-vctk-zero-tts-whisper-cer",
    "score": 0.0133294854,
    "raw_value": 0.013329485398892517,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/zonos/vctk/metrics.yaml (model_name Zonos-v0.1-transformer). Identity: Zyphra/Zonos-v0.1-transformer (notebooks/zonos.ipynb; hybrid line commented out). whisperv3_cer = 0.013329485398892517 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Zonos-v0.1-transformer",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/zonos.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "zonos-v0-1",
    "bench": "balacoon-vctk-zero-tts-redimnet-sim",
    "score": 0.7754620483,
    "raw_value": 0.775462048255034,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/vctk/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/zonos/vctk/metrics.yaml (model_name Zonos-v0.1-transformer). Identity: Zyphra/Zonos-v0.1-transformer (notebooks/zonos.ipynb; hybrid line commented out). redimnet_secs = 0.775462048255034 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Zonos-v0.1-transformer",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/vctk/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/zonos.ipynb"
    ]
  },
  {
    "model": "zonos-v0-1",
    "bench": "balacoon-daps-celeb-zero-tts-whisper-cer",
    "score": 0.0752882643,
    "raw_value": 0.07528826433984992,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/zonos/daps_celeb/metrics.yaml (model_name Zonos-v0.1-transformer). Identity: Zyphra/Zonos-v0.1-transformer (notebooks/zonos.ipynb; hybrid line commented out). whisperv3_cer = 0.07528826433984992 (openai/whisper-large-v3-turbo CER, fraction).",
    "source_model_label": "Zonos-v0.1-transformer",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/zonos.ipynb"
    ],
    "scale_divisor": 1
  },
  {
    "model": "zonos-v0-1",
    "bench": "balacoon-daps-celeb-zero-tts-redimnet-sim",
    "score": 0.6754473095,
    "raw_value": 0.6754473094956663,
    "source": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/daps_celeb/metrics.yaml",
    "source_id": "speech-balacoon-baselines-apache",
    "by": "independent",
    "date": "2025-02-18",
    "note": "zero-tts/zonos/daps_celeb/metrics.yaml (model_name Zonos-v0.1-transformer). Identity: Zyphra/Zonos-v0.1-transformer (notebooks/zonos.ipynb; hybrid line commented out). redimnet_secs = 0.6754473094956663 (ReDimNet speaker-embedding cosine, reference vs generated).",
    "source_model_label": "Zonos-v0.1-transformer",
    "identity_sources": [
      "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/zero-tts/zonos/daps_celeb/metrics.yaml",
      "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/notebooks/zonos.ipynb"
    ]
  },
  {
    "model": "voicecraft-830m-tts-enhanced",
    "bench": "librispeech-test-clean-zero-tts-hubert-wer",
    "score": 0.0468,
    "raw_value": 4.68,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section LibriSpeech test-clean, row 'VoiceCraft', WER column = 4.68 (HuBERT-large-ls960-ft WER). Identity: official pyp1/VoiceCraft 830M_TTSEnhanced.pth (paper Appendix D footnote 11). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "VoiceCraft",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "xtts-v2",
    "bench": "librispeech-test-clean-zero-tts-hubert-wer",
    "score": 0.042,
    "raw_value": 4.2,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section LibriSpeech test-clean, row 'XTTS-v2', WER column = 4.20 (HuBERT-large-ls960-ft WER). Identity: official coqui/XTTS-v2 checkpoint (paper Appendix D footnote 10). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "XTTS-v2",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "maskgct",
    "bench": "librispeech-test-clean-zero-tts-hubert-wer",
    "score": 0.02634,
    "raw_value": 2.634,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "lab",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section LibriSpeech test-clean, row 'MaskGCT', WER column = 2.634(1.976) (HuBERT-large-ls960-ft WER). Identity: own model amphion/MaskGCT (T2S-large, G2P, predicted total duration; the \"(rerank 5)\" parenthesised value and the \"gt length\" row are not used). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.04079,
    "raw_value": 4.079,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-en, row 'CosyVoice', WER column = 4.079 (Whisper-large-v3 WER). Identity: official model-scope/CosyVoice-300M (paper Appendix D footnote 12). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "xtts-v2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.03248,
    "raw_value": 3.248,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-en, row 'XTTS-v2', WER column = 3.248 (Whisper-large-v3 WER). Identity: official coqui/XTTS-v2 checkpoint (paper Appendix D footnote 10). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "XTTS-v2",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voicecraft-830m-tts-enhanced",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.07556,
    "raw_value": 7.556,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-en, row 'VoiceCraft', WER column = 7.556 (Whisper-large-v3 WER). Identity: official pyp1/VoiceCraft 830M_TTSEnhanced.pth (paper Appendix D footnote 11). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "VoiceCraft",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.02623,
    "raw_value": 2.623,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "lab",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-en, row 'MaskGCT', WER column = 2.623(1.283) (Whisper-large-v3 WER). Identity: own model amphion/MaskGCT (T2S-large, G2P, predicted total duration; the \"(rerank 5)\" parenthesised value and the \"gt length\" row are not used). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "cosyvoice-300m",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.04089,
    "raw_value": 4.089,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-zh, row 'CosyVoice', WER column = 4.089 (Paraformer-zh CER). Identity: official model-scope/CosyVoice-300M (paper Appendix D footnote 12). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "CosyVoice",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "xtts-v2",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.02876,
    "raw_value": 2.876,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "independent",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-zh, row 'XTTS-v2', WER column = 2.876 (Paraformer-zh CER). Identity: official coqui/XTTS-v2 checkpoint (paper Appendix D footnote 10). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "XTTS-v2",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.02273,
    "raw_value": 2.273,
    "source": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
    "source_id": "speech-amphion-maskgct-mit",
    "by": "lab",
    "date": "2024-10-20",
    "note": "MaskGCT README results table, section SeedTTS test-zh, row 'MaskGCT', WER column = 2.273(0.843) (Paraformer-zh CER). Identity: own model amphion/MaskGCT (T2S-large, G2P, predicted total duration; the \"(rerank 5)\" parenthesised value and the \"gt length\" row are not used). The MaskGCT paper (arXiv:2409.00750v3, used for provenance only) Appendix D states baselines were run with official code/checkpoints; VALL-E (Amphion reproduction), VoiceBox and NaturalSpeech 3 rows (scores copied from Ju et al.) are not imported. Table unchanged since README creation (commit bc1fe8a, 2024-10-20).",
    "source_model_label": "MaskGCT",
    "identity_sources": [
      "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md",
      "https://arxiv.org/abs/2409.00750v3"
    ],
    "scale_divisor": 100
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0324,
    "raw_value": 3.24,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '3.24(2.57)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: CosyVoice2-0.5B (cosyvoice2-vc config: FunAudioLLM/CosyVoice2-0.5B). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "CosyVoice2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0128,
    "raw_value": 1.28,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.28(1.45)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: CosyVoice2-0.5B (cosyvoice2-vc config: FunAudioLLM/CosyVoice2-0.5B). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "CosyVoice2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0214,
    "raw_value": 2.14,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '2.14(2.24)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: cosyvoice3-latest config: FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint with official CosyVoice code (not the _RL variant). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "CosyVoice3",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0114,
    "raw_value": 1.14,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.14(1.21)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: cosyvoice3-latest config: FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint with official CosyVoice code (not the _RL variant). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "CosyVoice3",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0208,
    "raw_value": 2.08,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '2.08(1.52)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: IndexTeam/IndexTTS-2. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "IndexTTS2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0104,
    "raw_value": 1.04,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.04(1.01)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: IndexTeam/IndexTTS-2. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "IndexTTS2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0181,
    "raw_value": 1.81,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.81(1.85)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: openbmb/VoxCPM-0.5B (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0104,
    "raw_value": 1.04,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.04(0.93)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: openbmb/VoxCPM-0.5B (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "VoxCPM",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.01643,
    "raw_value": 1.643,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.643 (1.84)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: openbmb/VoxCPM2, cfg_value=2.0, inference_timesteps=10, no denoise (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "VoxCPM2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0092,
    "raw_value": 0.92,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '0.920 (0.97)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: openbmb/VoxCPM2, cfg_value=2.0, inference_timesteps=10, no denoise (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "VoxCPM2",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0158,
    "raw_value": 1.58,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.58 (1.24)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (the xvec_only row is not used). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "qwen3_tts",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0087,
    "raw_value": 0.87,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '0.87 (0.78)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (the xvec_only row is not used). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "qwen3_tts",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0169,
    "raw_value": 1.69,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.69 (1.32)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "qwen3_tts",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0101,
    "raw_value": 1.01,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.01 (0.92)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "qwen3_tts",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0182,
    "raw_value": 1.82,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-04-01",
    "note": "UltraEval-Audio replication/fishaudio-s2-pro.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.82' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: open weights fishaudio/s2-pro via fish-speech local inference (doc notes paper numbers used the hosted service). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "fishaudio-s2-pro",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0103,
    "raw_value": 1.03,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-04-01",
    "note": "UltraEval-Audio replication/fishaudio-s2-pro.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.03' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: open weights fishaudio/s2-pro via fish-speech local inference (doc notes paper numbers used the hosted service). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "fishaudio-s2-pro",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100,
    "date_basis": "doc gives evaluation month only"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0269,
    "raw_value": 2.69,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '2.69(1.84)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), bfloat16. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "moss-tts-v1.5",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0149,
    "raw_value": 1.49,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.49(1.37)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), bfloat16. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "moss-tts-v1.5",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "fireredtts3-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0181,
    "raw_value": 1.81,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-08-22",
    "note": "UltraEval-Audio replication/FireRedTTS3.md, Seed-TTS-Eval table, test-en WER (openai/whisper-large-v3): printed '1.81 (1.64)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: FireRedTeam/FireRedTTS3, mode base (zero-shot cloning). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "FireRedTTS3",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "fireredtts3-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0095,
    "raw_value": 0.95,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2026-08-22",
    "note": "UltraEval-Audio replication/FireRedTTS3.md, Seed-TTS-Eval table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '0.95 (1.01)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: FireRedTeam/FireRedTTS3, mode base (zero-shot cloning). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_model_label": "FireRedTTS3",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "lmarena-text",
    "score": 1115.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'vicuna-7b'; 6,923 votes; published 95% interval 1106.1–1124.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "vicuna-13b-v1-5",
    "bench": "lmarena-text",
    "score": 1141.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'vicuna-13b'; 19,367 votes; published 95% interval 1134.8–1148.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "lmarena-text",
    "score": 1172.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'vicuna-33b'; 22,479 votes; published 95% interval 1166.7–1179.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "fastchat-t5-3b",
    "bench": "lmarena-text",
    "score": 992.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'fastchat-t5-3b'; 4,203 votes; published 95% interval 980.2–1004.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "alpaca-13b",
    "bench": "lmarena-text",
    "score": 1070.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'alpaca-13b'; 5,745 votes; published 95% interval 1058.7–1081.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "koala-13b",
    "bench": "lmarena-text",
    "score": 1070.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'koala-13b'; 6,965 votes; published 95% interval 1061.0–1080.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "lmarena-text",
    "score": 1023.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'oasst-pythia-12b'; 6,310 votes; published 95% interval 1012.6–1034.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt4all-13b-snoozy",
    "bench": "lmarena-text",
    "score": 1067.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt4all-13b-snoozy'; 1,743 votes; published 95% interval 1052.5–1082.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "guanaco-33b",
    "bench": "lmarena-text",
    "score": 1127.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'guanaco-33b'; 2,921 votes; published 95% interval 1115.4–1139.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "rwkv-4-raven-14b",
    "bench": "lmarena-text",
    "score": 1042.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'RWKV-4-Raven-14B'; 4,845 votes; published 95% interval 1030.9–1053.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "stablelm-tuned-alpha-7b",
    "bench": "lmarena-text",
    "score": 953.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'stablelm-tuned-alpha-7b'; 3,287 votes; published 95% interval 940.5–965.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "dolly-v2-12b",
    "bench": "lmarena-text",
    "score": 982.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'dolly-v2-12b'; 3,412 votes; published 95% interval 969.0–995.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mpt-7b-chat",
    "bench": "lmarena-text",
    "score": 1063.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mpt-7b-chat'; 3,924 votes; published 95% interval 1051.5–1075.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mpt-30b-chat",
    "bench": "lmarena-text",
    "score": 1150.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mpt-30b-chat'; 2,572 votes; published 95% interval 1138.5–1162.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "falcon-180b-chat",
    "bench": "lmarena-text",
    "score": 1148.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'falcon-180b-chat'; 1,295 votes; published 95% interval 1131.0–1165.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "dolphin-2-2-1-mistral-7b",
    "bench": "lmarena-text",
    "score": 1152.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'dolphin-2.2.1-mistral-7b'; 1,679 votes; published 95% interval 1136.9–1167.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "lmarena-text",
    "score": 1175.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'openhermes-2.5-mistral-7b'; 5,006 votes; published 95% interval 1165.5–1186.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nous-hermes-2-mixtral-8x7b-dpo",
    "bench": "lmarena-text",
    "score": 1164.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nous-hermes-2-mixtral-8x7b-dpo'; 3,777 votes; published 95% interval 1152.5–1176.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "openchat-3-5",
    "bench": "lmarena-text",
    "score": 1183,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'openchat-3.5'; 7,968 votes; published 95% interval 1173.3–1192.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "lmarena-text",
    "score": 1182.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'openchat-3.5-0106'; 12,637 votes; published 95% interval 1174.7–1190.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "lmarena-text",
    "score": 1167.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'starling-lm-7b-alpha'; 10,224 votes; published 95% interval 1159.3–1175.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "lmarena-text",
    "score": 1170.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'starling-lm-7b-beta'; 16,056 votes; published 95% interval 1163.5–1178.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "lmarena-text",
    "score": 1126.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'zephyr-7b-alpha'; 1,785 votes; published 95% interval 1111.3–1142.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "lmarena-text",
    "score": 1130.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'zephyr-7b-beta'; 11,118 votes; published 95% interval 1121.9–1139.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "zephyr-orpo-141b-a35b",
    "bench": "lmarena-text",
    "score": 1213,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'zephyr-orpo-141b-A35b-v0.1'; 4,652 votes; published 95% interval 1202.2–1223.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "stripedhyena-nous-7b",
    "bench": "lmarena-text",
    "score": 1121.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'stripedhyena-nous-7b'; 5,182 votes; published 95% interval 1110.6–1132.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "lmarena-text",
    "score": 1152.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'solar-10.7b-instruct-v1.0'; 4,155 votes; published 95% interval 1139.2–1165.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "smollm2-1-7b",
    "bench": "lmarena-text",
    "score": 1114.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'smollm2-1.7b-instruct'; 2,199 votes; published 95% interval 1100.3–1129.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "snowflake-arctic-instruct",
    "bench": "lmarena-text",
    "score": 1180,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'snowflake-arctic-instruct'; 32,832 votes; published 95% interval 1174.1–1185.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-2-9b-simpo",
    "bench": "lmarena-text",
    "score": 1280.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-2-9b-it-simpo'; 10,072 votes; published 95% interval 1273.5–1287.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "athene-70b",
    "bench": "lmarena-text",
    "score": 1306.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'athene-70b-0725'; 19,621 votes; published 95% interval 1300.9–1312.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "athene-v2-chat",
    "bench": "lmarena-text",
    "score": 1314.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'athene-v2-chat'; 24,739 votes; published 95% interval 1309.6–1318.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "lmarena-text",
    "score": 1177.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'tulu-2-dpo-70b'; 6,535 votes; published 95% interval 1167.8–1187.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "lmarena-text",
    "score": 1190.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'internlm2_5-20b-chat'; 9,901 votes; published 95% interval 1183.5–1197.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "chatglm-6b",
    "bench": "lmarena-text",
    "score": 995.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'chatglm-6b'; 4,914 votes; published 95% interval 983.3–1008.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "chatglm2-6b",
    "bench": "lmarena-text",
    "score": 1024.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'chatglm2-6b'; 2,658 votes; published 95% interval 1011.2–1038.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "chatglm3-6b",
    "bench": "lmarena-text",
    "score": 1056.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'chatglm3-6b'; 4,658 votes; published 95% interval 1044.9–1068.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "cogvlm2-llama3-chat-19b",
    "bench": "lmarena-vision",
    "score": 963.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'cogvlm2-llama3-chat-19b'; 1,991 votes; published 95% interval 948.5–979.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-14b-chat",
    "bench": "lmarena-text",
    "score": 1139.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen-14b-chat'; 4,964 votes; published 95% interval 1128.4–1150.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-4b-chat",
    "bench": "lmarena-text",
    "score": 1091.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-4b-chat'; 7,597 votes; published 95% interval 1081.8–1100.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "lmarena-text",
    "score": 1143.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-7b-chat'; 4,737 votes; published 95% interval 1133.9–1153.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "lmarena-text",
    "score": 1191,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-14b-chat'; 17,839 votes; published 95% interval 1183.9–1198.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "lmarena-text",
    "score": 1203.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-32b-chat'; 21,741 votes; published 95% interval 1197.8–1210.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "lmarena-text",
    "score": 1233.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-72b-chat'; 39,302 votes; published 95% interval 1228.2–1238.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "lmarena-text",
    "score": 1234.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen1.5-110b-chat'; 26,195 votes; published 95% interval 1228.6–1239.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-72b",
    "bench": "lmarena-text",
    "score": 1261.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen2-72b-instruct'; 37,325 votes; published 95% interval 1256.8–1266.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwq-32b",
    "bench": "lmarena-text",
    "score": 1335.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwq-32b'; 25,078 votes; published 95% interval 1331.3–1340.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwq-32b-preview",
    "bench": "lmarena-text",
    "score": 1154.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwq-32b-preview'; 3,231 votes; published 95% interval 1142.7–1165.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "lmarena-text",
    "score": 1326.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-30b-a3b'; 26,037 votes; published 95% interval 1321.7–1331.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "lmarena-vision",
    "score": 1030.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen2-vl-7b-instruct'; 5,766 votes; published 95% interval 1020.3–1041.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "lmarena-vision",
    "score": 1084.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen2-vl-72b'; 5,937 votes; published 95% interval 1074.6–1094.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "lmarena-vision",
    "score": 1118.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen2.5-vl-32b-instruct'; 1,490 votes; published 95% interval 1103.1–1134.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "lmarena-vision",
    "score": 1121.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen2.5-vl-72b-instruct'; 3,768 votes; published 95% interval 1110.3–1132.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "lmarena-text",
    "score": 1220.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-tulu-3-8b'; 2,896 votes; published 95% interval 1209.8–1231.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "lmarena-text",
    "score": 1286.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-tulu-3-70b'; 2,846 votes; published 95% interval 1275.6–1296.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "olmo-2-32b",
    "bench": "lmarena-text",
    "score": 1251.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'olmo-2-0325-32b-instruct'; 3,334 votes; published 95% interval 1240.9–1262.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "lmarena-text",
    "score": 1073.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'olmo-7b-instruct'; 6,328 votes; published 95% interval 1062.4–1084.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "molmo-7b-d",
    "bench": "lmarena-vision",
    "score": 993.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'molmo-7b-d-0924'; 2,815 votes; published 95% interval 980.0–1007.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "molmo-72b",
    "bench": "lmarena-vision",
    "score": 1044.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'molmo-72b-0924'; 3,048 votes; published 95% interval 1031.8–1058.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-2b",
    "bench": "lmarena-text",
    "score": 1093.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-2b-it'; 4,780 votes; published 95% interval 1082.1–1105.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-7b",
    "bench": "lmarena-text",
    "score": 1137.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-7b-it'; 8,925 votes; published 95% interval 1128.2–1147.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "lmarena-text",
    "score": 1116.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-1.1-2b-it'; 10,854 votes; published 95% interval 1108.9–1124.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "lmarena-text",
    "score": 1182.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-1.1-7b-it'; 23,893 votes; published 95% interval 1176.4–1188.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-2-2b",
    "bench": "lmarena-text",
    "score": 1200,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-2-2b-it'; 46,616 votes; published 95% interval 1195.9–1204.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-2-9b",
    "bench": "lmarena-text",
    "score": 1267,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-2-9b-it'; 54,611 votes; published 95% interval 1263.1–1270.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemma-2-27b",
    "bench": "lmarena-text",
    "score": 1289.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemma-2-27b-it'; 75,754 votes; published 95% interval 1286.0–1292.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-3-0-2b",
    "bench": "lmarena-text",
    "score": 1156.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-3.0-2b-instruct'; 6,837 votes; published 95% interval 1148.2–1165.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-3-0-8b",
    "bench": "lmarena-text",
    "score": 1183.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-3.0-8b-instruct'; 6,638 votes; published 95% interval 1174.4–1191.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-3-1-2b",
    "bench": "lmarena-text",
    "score": 1178.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-3.1-2b-instruct'; 3,188 votes; published 95% interval 1167.6–1190.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "granite-3-1-8b",
    "bench": "lmarena-text",
    "score": 1208.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'granite-3.1-8b-instruct'; 3,090 votes; published 95% interval 1197.4–1219.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "lmarena-text",
    "score": 1107.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-2-7b-chat'; 14,148 votes; published 95% interval 1100.8–1114.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "lmarena-text",
    "score": 1141.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-2-13b-chat'; 19,174 votes; published 95% interval 1134.6–1148.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "lmarena-text",
    "score": 1170.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-2-70b-chat'; 38,492 votes; published 95% interval 1165.1–1176.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-8b",
    "bench": "lmarena-text",
    "score": 1223.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3-8b-instruct'; 104,642 votes; published 95% interval 1219.8–1227.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-70b",
    "bench": "lmarena-text",
    "score": 1276.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3-70b-instruct'; 156,876 votes; published 95% interval 1272.9–1280.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "lmarena-text",
    "score": 1136.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'codellama-34b-instruct'; 7,366 votes; published 95% interval 1127.9–1145.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "lmarena-text",
    "score": 1119.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'codellama-70b-instruct'; 1,143 votes; published 95% interval 1101.0–1137.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "lmarena-vision",
    "score": 992.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llama-3.2-vision-11b-instruct'; 4,817 votes; published 95% interval 981.4–1004.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "lmarena-vision",
    "score": 1032.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llama-3.2-vision-90b-instruct'; 8,682 votes; published 95% interval 1023.1–1041.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-mini-4k-instruct",
    "bench": "lmarena-text",
    "score": 1128.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-3-mini-4k-instruct'; 20,118 votes; published 95% interval 1121.7–1134.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "lmarena-text",
    "score": 1143.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-3-mini-4k-instruct-june-2024'; 12,297 votes; published 95% interval 1136.6–1149.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "lmarena-text",
    "score": 1129.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-3-mini-128k-instruct'; 20,685 votes; published 95% interval 1122.4–1137.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-small-8k-instruct",
    "bench": "lmarena-text",
    "score": 1171,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-3-small-8k-instruct'; 17,766 votes; published 95% interval 1165.1–1176.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "lmarena-text",
    "score": 1197.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'phi-3-medium-4k-instruct'; 25,055 votes; published 95% interval 1192.7–1203.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "lmarena-vision",
    "score": 881.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'phi-3-vision-128k-instruct'; 1,401 votes; published 95% interval 862.4–899.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "lmarena-vision",
    "score": 919.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'phi-3.5-vision-instruct'; 2,592 votes; published 95% interval 903.7–935.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "lmarena-text",
    "score": 1149.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'wizardlm-13b'; 7,044 votes; published 95% interval 1140.3–1158.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "lmarena-text",
    "score": 1184.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'wizardlm-70b'; 8,214 votes; published 95% interval 1175.3–1194.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "lmarena-text",
    "score": 1286.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-nemotron-51b-instruct'; 3,749 votes; published 95% interval 1276.6–1296.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "lmarena-text",
    "score": 1298.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.1-nemotron-70b-instruct'; 7,140 votes; published 95% interval 1291.0–1306.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "lmarena-text",
    "score": 1327.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama-3.3-nemotron-49b-super-v1'; 2,218 votes; published 95% interval 1315.7–1339.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama2-70b-steerlm-chat",
    "bench": "lmarena-text",
    "score": 1154.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'llama2-70b-steerlm-chat'; 3,585 votes; published 95% interval 1142.1–1167.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nemotron-4-340b-instruct",
    "bench": "lmarena-text",
    "score": 1277.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'nemotron-4-340b-instruct'; 19,659 votes; published 95% interval 1271.8–1282.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "internvl2-4b",
    "bench": "lmarena-vision",
    "score": 958.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'internvl2-4b'; 3,703 votes; published 95% interval 945.7–970.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "internvl2-26b",
    "bench": "lmarena-vision",
    "score": 1024.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'internvl2-26b'; 5,148 votes; published 95% interval 1012.0–1037.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llava-onevision-qwen2-72b-ov",
    "bench": "lmarena-vision",
    "score": 978.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llava-onevision-qwen2-72b-ov'; 1,321 votes; published 95% interval 960.1–996.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "lmarena-vision",
    "score": 964.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'llava-v1.6-34b'; 4,531 votes; published 95% interval 952.1–976.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "minicpm-v-2-6",
    "bench": "lmarena-vision",
    "score": 962.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'minicpm-v-2_6'; 1,987 votes; published 95% interval 946.8–978.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "lmarena-text",
    "score": 1110.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-7b-instruct'; 8,977 votes; published 95% interval 1101.0–1119.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "lmarena-text",
    "score": 1149.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-7b-instruct-v0.2'; 19,402 votes; published 95% interval 1142.7–1156.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "lmarena-text",
    "score": 1197.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mixtral-8x7b-instruct-v0.1'; 73,503 votes; published 95% interval 1192.9–1201.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "lmarena-text",
    "score": 1229.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mixtral-8x22b-instruct-v0.1'; 51,416 votes; published 95% interval 1225.0–1234.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-small-3",
    "bench": "lmarena-text",
    "score": 1274.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-small-24b-instruct-2501'; 14,681 votes; published 95% interval 1268.5–1280.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "lmarena-text",
    "score": 1303.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-small-3.1-24b-instruct-2503'; 32,551 votes; published 95% interval 1298.6–1307.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "lmarena-vision",
    "score": 1127.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-small-3.1-24b-instruct-2503'; 28,504 votes; published 95% interval 1118.6–1135.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "lmarena-text",
    "score": 1237.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ministral-8b-2410'; 4,781 votes; published 95% interval 1228.5–1246.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pixtral-12b",
    "bench": "lmarena-vision",
    "score": 1025.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'pixtral-12b-2409'; 7,511 votes; published 95% interval 1015.6–1034.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "pixtral-large",
    "bench": "lmarena-vision",
    "score": 1095.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'pixtral-large-2411'; 5,423 votes; published 95% interval 1085.9–1104.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-2407",
    "bench": "lmarena-text",
    "score": 1314.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-large-2407'; 45,459 votes; published 95% interval 1310.4–1318.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "command-r-03-2024",
    "bench": "lmarena-text",
    "score": 1226.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'command-r'; 54,036 votes; published 95% interval 1222.0–1231.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "command-r-08-2024",
    "bench": "lmarena-text",
    "score": 1250.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'command-r-08-2024'; 10,140 votes; published 95% interval 1243.8–1257.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "lmarena-text",
    "score": 1261.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'command-r-plus'; 77,554 votes; published 95% interval 1257.4–1266.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "lmarena-text",
    "score": 1276.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'command-r-plus-08-2024'; 9,866 votes; published 95% interval 1269.7–1282.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "aya-expanse-8b",
    "bench": "lmarena-text",
    "score": 1223,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'c4ai-aya-expanse-8b'; 9,818 votes; published 95% interval 1216.0–1230.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "aya-expanse-32b",
    "bench": "lmarena-text",
    "score": 1267.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'c4ai-aya-expanse-32b'; 27,124 votes; published 95% interval 1262.3–1272.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "aya-vision-32b",
    "bench": "lmarena-vision",
    "score": 1000.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'c4ai-aya-vision-32b'; 847 votes; published 95% interval 978.2–1022.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "lmarena-text",
    "score": 1225.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-3.5-turbo-0125'; 66,207 votes; published 95% interval 1220.8–1230.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "lmarena-text",
    "score": 1204.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-3.5-turbo-1106'; 16,619 votes; published 95% interval 1195.4–1212.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-0314",
    "bench": "lmarena-text",
    "score": 1287.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4-0314'; 54,173 votes; published 95% interval 1283.0–1292.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-0613",
    "bench": "lmarena-text",
    "score": 1276.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4-0613'; 88,723 votes; published 95% interval 1272.1–1280.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "lmarena-text",
    "score": 1312.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4-1106-preview'; 100,105 votes; published 95% interval 1308.9–1316.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "lmarena-text",
    "score": 1313.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4-0125-preview'; 93,439 votes; published 95% interval 1309.1–1317.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "lmarena-text",
    "score": 1324.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4-turbo-2024-04-09'; 98,114 votes; published 95% interval 1320.5–1328.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "lmarena-vision",
    "score": 1112,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4-turbo-2024-04-09'; 13,391 votes; published 95% interval 1100.4–1123.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "lmarena-text",
    "score": 1346.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4o-2024-05-13'; 112,881 votes; published 95% interval 1342.6–1349.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "lmarena-vision",
    "score": 1161.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4o-2024-05-13'; 23,273 votes; published 95% interval 1153.1–1170.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "lmarena-text",
    "score": 1335.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4o-2024-08-06'; 45,499 votes; published 95% interval 1331.3–1339.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "lmarena-vision",
    "score": 1118.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4o-2024-08-06'; 3,376 votes; published 95% interval 1106.0–1130.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "lmarena-text",
    "score": 1317.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4o-mini-2024-07-18'; 68,697 votes; published 95% interval 1314.1–1321.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "lmarena-vision",
    "score": 1097.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4o-mini-2024-07-18'; 17,341 votes; published 95% interval 1088.8–1105.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "chatgpt-4o-latest-20250326",
    "bench": "lmarena-text",
    "score": 1442.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'chatgpt-4o-latest-20250326'; 81,595 votes; published 95% interval 1440.1–1445.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "chatgpt-4o-latest-20250326",
    "bench": "lmarena-vision",
    "score": 1238.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'chatgpt-4o-latest-20250326'; 23,416 votes; published 95% interval 1233.3–1244.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "lmarena-text",
    "score": 1322.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4.1-nano-2025-04-14'; 6,103 votes; published 95% interval 1314.6–1330.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "lmarena-vision",
    "score": 1088.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4.1-nano-2025-04-14'; 1,211 votes; published 95% interval 1070.4–1107.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "lmarena-text",
    "score": 1444.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-4.5-preview-2025-02-27'; 14,547 votes; published 95% interval 1439.0–1450.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "lmarena-vision",
    "score": 1225.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-4.5-preview-2025-02-27'; 2,925 votes; published 95% interval 1213.4–1236.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o1-2024-12-17",
    "bench": "lmarena-text",
    "score": 1402.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o1-2024-12-17'; 27,807 votes; published 95% interval 1397.8–1406.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o1-2024-12-17",
    "bench": "lmarena-vision",
    "score": 1192.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'o1-2024-12-17'; 3,694 votes; published 95% interval 1182.0–1203.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o1-mini",
    "bench": "lmarena-text",
    "score": 1336.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o1-mini'; 51,981 votes; published 95% interval 1333.2–1340.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o1-preview",
    "bench": "lmarena-text",
    "score": 1388.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o1-preview'; 31,122 votes; published 95% interval 1383.5–1393.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "o3-mini",
    "bench": "lmarena-text",
    "score": 1363.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'o3-mini-high'; 18,589 votes; published 95% interval 1358.3–1368.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2-chat-latest-20260210",
    "bench": "lmarena-text",
    "score": 1475.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.2-chat-latest-20260210'; 35,937 votes; published 95% interval 1471.7–1479.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2-chat-latest-20260210",
    "bench": "lmarena-vision",
    "score": 1278,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5.2-chat-latest-20260210'; 16,480 votes; published 95% interval 1271.5–1284.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1-codex",
    "bench": "lmarena-webdev",
    "score": 1337.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.1-codex'; 5,005 votes; published 95% interval 1325.6–1348.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-1-codex-mini",
    "bench": "lmarena-webdev",
    "score": 1243.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.1-codex-mini'; 1,155 votes; published 95% interval 1221.6–1265.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-2-codex",
    "bench": "lmarena-webdev",
    "score": 1338.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-5.2-codex'; 6,722 votes; published 95% interval 1330.0–1347.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "lmarena-text",
    "score": 1322.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-opus-20240229'; 194,909 votes; published 95% interval 1319.0–1325.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "lmarena-vision",
    "score": 1061.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-opus-20240229'; 15,565 votes; published 95% interval 1051.3–1072.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "lmarena-text",
    "score": 1281.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-sonnet-20240229'; 109,284 votes; published 95% interval 1277.3–1285.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "lmarena-vision",
    "score": 1016.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-sonnet-20240229'; 12,314 votes; published 95% interval 1005.0–1027.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "lmarena-text",
    "score": 1261.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-haiku-20240307'; 117,701 votes; published 95% interval 1257.8–1265.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "lmarena-vision",
    "score": 1000,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-haiku-20240307'; 13,380 votes; published 95% interval 987.5–1012.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "lmarena-text",
    "score": 1343.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-5-sonnet-20240620'; 82,419 votes; published 95% interval 1339.7–1346.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "lmarena-vision",
    "score": 1145.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-5-sonnet-20240620'; 21,624 votes; published 95% interval 1135.1–1155.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "lmarena-text",
    "score": 1374,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-5-sonnet-20241022'; 87,638 votes; published 95% interval 1370.8–1377.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "lmarena-vision",
    "score": 1160.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-5-sonnet-20241022'; 10,374 votes; published 95% interval 1152.8–1168.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "lmarena-text",
    "score": 1324.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-5-haiku-20241022'; 68,873 votes; published 95% interval 1321.3–1327.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "lmarena-vision",
    "score": 1126.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-5-haiku-20241022'; 1,400 votes; published 95% interval 1110.7–1142.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "lmarena-text",
    "score": 1388.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-3-7-sonnet-20250219-thinking-32k'; 38,033 votes; published 95% interval 1384.1–1392.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "lmarena-vision",
    "score": 1192.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-3-7-sonnet-20250219-thinking-32k'; 1,438 votes; published 95% interval 1177.3–1208.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "lmarena-text",
    "score": 1401.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-sonnet-4-20250514-thinking-32k'; 33,913 votes; published 95% interval 1397.1–1406.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "lmarena-vision",
    "score": 1206.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-sonnet-4-20250514-thinking-32k'; 1,145 votes; published 95% interval 1189.3–1223.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "lmarena-text",
    "score": 1426.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-20250514-thinking-16k'; 35,691 votes; published 95% interval 1421.7–1430.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "lmarena-vision",
    "score": 1207.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-opus-4-20250514-thinking-16k'; 1,290 votes; published 95% interval 1191.3–1223.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "lmarena-search",
    "score": 1150.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-search'; 31,037 votes; published 95% interval 1145.1–1156.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "lmarena-text",
    "score": 1449.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-opus-4-1-20250805-thinking-16k'; 49,306 votes; published 95% interval 1446.3–1453.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "lmarena-webdev",
    "score": 1389.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-opus-4-1-20250805'; 6,926 votes; published 95% interval 1378.7–1400.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "lmarena-search",
    "score": 1167.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'claude-opus-4-1-search'; 76,933 votes; published 95% interval 1162.1–1172.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "lmarena-text",
    "score": 1236.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-pro-dev-api'; 18,354 votes; published 95% interval 1229.5–1244.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "lmarena-text",
    "score": 1324,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-1.5-pro-001'; 79,138 votes; published 95% interval 1320.0–1328.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "lmarena-vision",
    "score": 1119.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-1.5-pro-001'; 16,734 votes; published 95% interval 1108.0–1131.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "lmarena-text",
    "score": 1351.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-1.5-pro-002'; 55,606 votes; published 95% interval 1347.9–1354.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "lmarena-vision",
    "score": 1179.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-1.5-pro-002'; 8,902 votes; published 95% interval 1170.4–1188.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "lmarena-text",
    "score": 1286.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-001'; 62,833 votes; published 95% interval 1282.0–1291.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "lmarena-vision",
    "score": 1059.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-001'; 13,260 votes; published 95% interval 1047.6–1071.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "lmarena-text",
    "score": 1309.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-002'; 34,902 votes; published 95% interval 1305.0–1313.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "lmarena-vision",
    "score": 1141.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-002'; 7,241 votes; published 95% interval 1131.5–1151.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-8b-001",
    "bench": "lmarena-text",
    "score": 1258.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-8b-001'; 35,558 votes; published 95% interval 1254.4–1263.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-1-5-flash-8b-001",
    "bench": "lmarena-vision",
    "score": 1071.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-1.5-flash-8b-001'; 6,243 votes; published 95% interval 1060.7–1081.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "lmarena-text",
    "score": 1359.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.0-flash-001'; 43,310 votes; published 95% interval 1356.1–1363.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "lmarena-vision",
    "score": 1171.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.0-flash-001'; 10,134 votes; published 95% interval 1164.6–1179.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-lite-preview-02-05",
    "bench": "lmarena-text",
    "score": 1353.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.0-flash-lite-preview-02-05'; 24,955 votes; published 95% interval 1349.2–1357.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-0-flash-lite-preview-02-05",
    "bench": "lmarena-vision",
    "score": 1136,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.0-flash-lite-preview-02-05'; 3,991 votes; published 95% interval 1125.5–1146.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "lmarena-text",
    "score": 1403.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-preview-09-2025'; 32,839 votes; published 95% interval 1399.1–1407.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "lmarena-vision",
    "score": 1230.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-preview-09-2025'; 4,794 votes; published 95% interval 1220.5–1240.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-lite-preview-09-2025",
    "bench": "lmarena-text",
    "score": 1379.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-lite-preview-09-2025-no-thinking'; 47,254 votes; published 95% interval 1376.0–1382.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-2-5-flash-lite-preview-09-2025",
    "bench": "lmarena-vision",
    "score": 1174.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-2.5-flash-lite-preview-09-2025-no-thinking'; 4,862 votes; published 95% interval 1164.9–1184.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "palm-2-chat-bison",
    "bench": "lmarena-text",
    "score": 1139.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'palm-2'; 8,554 votes; published 95% interval 1129.8–1148.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-2-0813",
    "bench": "lmarena-text",
    "score": 1335.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-2-2024-08-13'; 63,498 votes; published 95% interval 1331.9–1339.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-2-mini-0813",
    "bench": "lmarena-text",
    "score": 1308.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-2-mini-2024-08-13'; 52,567 votes; published 95% interval 1304.6–1312.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-3-preview-02-24",
    "bench": "lmarena-text",
    "score": 1411.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-3-preview-02-24'; 32,386 votes; published 95% interval 1407.1–1415.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "lmarena-text",
    "score": 1363.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-3-mini-high'; 15,505 votes; published 95% interval 1358.0–1369.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-0709",
    "bench": "lmarena-text",
    "score": 1410.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4-0709'; 39,398 votes; published 95% interval 1406.8–1414.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-0709",
    "bench": "lmarena-vision",
    "score": 1184.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4-0709'; 30,184 votes; published 95% interval 1176.9–1191.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-0709",
    "bench": "lmarena-search",
    "score": 1121.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4-search'; 19,108 votes; published 95% interval 1115.4–1127.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-fast",
    "bench": "lmarena-text",
    "score": 1404.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4-fast-reasoning'; 18,142 votes; published 95% interval 1399.8–1410.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-fast",
    "bench": "lmarena-webdev",
    "score": 1158.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4-fast-reasoning'; 742 votes; published 95% interval 1131.5–1186.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-fast",
    "bench": "lmarena-search",
    "score": 1160.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4-fast-search'; 41,794 votes; published 95% interval 1155.5–1165.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1",
    "bench": "lmarena-text",
    "score": 1465.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.1-thinking'; 66,366 votes; published 95% interval 1462.0–1468.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1",
    "bench": "lmarena-webdev",
    "score": 1213.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4.1-thinking'; 961 votes; published 95% interval 1188.9–1238.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20-multi-agent-0309",
    "bench": "lmarena-text",
    "score": 1470.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4.20-multi-agent-beta-0309'; 64,788 votes; published 95% interval 1466.9–1474.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20-multi-agent-0309",
    "bench": "lmarena-vision",
    "score": 1250.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4.20-multi-agent-beta-0309'; 24,571 votes; published 95% interval 1244.2–1256.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-20-multi-agent-0309",
    "bench": "lmarena-search",
    "score": 1204.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4.20-multi-agent-beta-0309'; 109,553 votes; published 95% interval 1199.5–1210.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-code-fast-1",
    "bench": "lmarena-webdev",
    "score": 1167,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-code-fast-1'; 789 votes; published 95% interval 1138.9–1195.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-lite",
    "bench": "lmarena-text",
    "score": 1260.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-lite-v1.0'; 19,372 votes; published 95% interval 1255.2–1265.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-lite",
    "bench": "lmarena-vision",
    "score": 1018.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'amazon-nova-lite-v1.0'; 1,854 votes; published 95% interval 1003.7–1033.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-pro",
    "bench": "lmarena-text",
    "score": 1290.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-pro-v1.0'; 24,745 votes; published 95% interval 1285.5–1294.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "nova-pro",
    "bench": "lmarena-vision",
    "score": 1018.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'amazon-nova-pro-v1.0'; 2,335 votes; published 95% interval 1004.9–1032.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-2312",
    "bench": "lmarena-text",
    "score": 1222.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-medium'; 34,550 votes; published 95% interval 1217.2–1228.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-large-2402",
    "bench": "lmarena-text",
    "score": 1242.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-large-2402'; 62,436 votes; published 95% interval 1237.8–1247.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-2505",
    "bench": "lmarena-text",
    "score": 1387.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-medium-2505'; 32,615 votes; published 95% interval 1382.8–1392.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-2505",
    "bench": "lmarena-vision",
    "score": 1155.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-medium-2505'; 10,809 votes; published 95% interval 1146.9–1163.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-3-1",
    "bench": "lmarena-text",
    "score": 1407.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mistral-medium-2508'; 94,966 votes; published 95% interval 1405.3–1410.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mistral-medium-3-1",
    "bench": "lmarena-vision",
    "score": 1159.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mistral-medium-2508'; 41,639 votes; published 95% interval 1153.1–1165.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "magistral-medium-2506",
    "bench": "lmarena-text",
    "score": 1304.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'magistral-medium-2506'; 11,432 votes; published 95% interval 1298.3–1311.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "devstral-medium-2507",
    "bench": "lmarena-webdev",
    "score": 1075.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'devstral-medium-2507'; 842 votes; published 95% interval 1045.2–1106.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-max-2024-09-19",
    "bench": "lmarena-text",
    "score": 1318.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen-max-0919'; 16,478 votes; published 95% interval 1312.4–1323.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-plus-2025-01-25",
    "bench": "lmarena-text",
    "score": 1345.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen-plus-0125'; 5,819 votes; published 95% interval 1337.6–1354.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "lmarena-text",
    "score": 1373.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen2.5-max'; 32,411 votes; published 95% interval 1369.7–1377.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-plus-2024-11-27",
    "bench": "lmarena-text",
    "score": 1314.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen2.5-plus-1127'; 10,187 votes; published 95% interval 1308.3–1321.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-max-preview",
    "bench": "lmarena-text",
    "score": 1434.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-max-preview'; 27,232 votes; published 95% interval 1429.9–1439.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "lmarena-text",
    "score": 1422.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-max-2025-09-23'; 8,969 votes; published 95% interval 1416.4–1429.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-flash",
    "bench": "lmarena-text",
    "score": 1395.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-flash'; 61,669 votes; published 95% interval 1392.3–1399.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-flash",
    "bench": "lmarena-webdev",
    "score": 1243.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.5-flash'; 1,329 votes; published 95% interval 1223.5–1262.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-5-max-preview",
    "bench": "lmarena-text",
    "score": 1464.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.5-max-preview'; 22,555 votes; published 95% interval 1459.8–1469.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "lmarena-text",
    "score": 1460.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3.6-max-preview'; 5,406 votes; published 95% interval 1452.1–1468.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-6-max-preview",
    "bench": "lmarena-webdev",
    "score": 1481.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'qwen3.6-max-preview'; 2,716 votes; published 95% interval 1469.0–1494.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-vl-max-2024-11-19",
    "bench": "lmarena-vision",
    "score": 1084.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen-vl-max-1119'; 1,422 votes; published 95% interval 1068.6–1100.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-vl-max-2025-08-13",
    "bench": "lmarena-vision",
    "score": 1184.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen-vl-max-2025-08-13'; 3,173 votes; published 95% interval 1172.8–1197.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen-image-2-0-pro-2026-06-22",
    "bench": "lmarena-image",
    "score": 1191.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'qwen-image-2.0-pro-2026-06-22'; 13,096 votes; published 95% interval 1185.9–1197.6; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1191.7), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-image-2-0-pro-2026-06-22",
    "bench": "lmarena-image-edit",
    "score": 1303.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'qwen-image-2.0-pro-2026-06-22'; 56,723 votes; published 95% interval 1298.5–1307.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1303.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "glm-4-0520",
    "bench": "lmarena-text",
    "score": 1273.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4-0520'; 9,788 votes; published 95% interval 1266.4–1280.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-plus",
    "bench": "lmarena-text",
    "score": 1319.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4-plus'; 26,126 votes; published 95% interval 1314.4–1324.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-large-2025-02-10",
    "bench": "lmarena-text",
    "score": 1326.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-large-2025-02-10'; 3,738 votes; published 95% interval 1316.6–1336.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-standard-2025-02-10",
    "bench": "lmarena-text",
    "score": 1311.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-standard-2025-02-10'; 3,904 votes; published 95% interval 1301.6–1320.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-t1-20250711",
    "bench": "lmarena-text",
    "score": 1387.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-t1-20250711'; 4,572 votes; published 95% interval 1378.5–1395.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-turbos-20250226",
    "bench": "lmarena-text",
    "score": 1348.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-turbos-20250226'; 2,220 votes; published 95% interval 1337.2–1360.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-turbos-20250416",
    "bench": "lmarena-text",
    "score": 1382.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-turbos-20250416'; 10,577 votes; published 95% interval 1376.2–1389.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-large-vision",
    "bench": "lmarena-text",
    "score": 1294,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-large-vision'; 5,248 votes; published 95% interval 1284.8–1303.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-large-vision",
    "bench": "lmarena-vision",
    "score": 1148.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'hunyuan-large-vision'; 1,274 votes; published 95% interval 1132.3–1165.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-standard-vision-2024-12-31",
    "bench": "lmarena-vision",
    "score": 1041.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'hunyuan-standard-vision-2024-12-31'; 809 votes; published 95% interval 1020.7–1062.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-vision-1-5-thinking",
    "bench": "lmarena-text",
    "score": 1394.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-vision-1.5-thinking'; 2,158 votes; published 95% interval 1382.5–1407.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-vision-1-5-thinking",
    "bench": "lmarena-vision",
    "score": 1155.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'hunyuan-vision-1.5-thinking'; 2,429 votes; published 95% interval 1143.4–1167.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-0-0110",
    "bench": "lmarena-text",
    "score": 1446.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ernie-5.0-0110'; 36,413 votes; published 95% interval 1442.5–1450.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-0-preview-1022",
    "bench": "lmarena-text",
    "score": 1418.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ernie-5.0-preview-1022'; 4,724 votes; published 95% interval 1409.9–1427.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-0-preview-1203",
    "bench": "lmarena-text",
    "score": 1448.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ernie-5.0-preview-1203'; 9,788 votes; published 95% interval 1442.3–1455.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ernie-5-0-preview-1220",
    "bench": "lmarena-vision",
    "score": 1218.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'ernie-5.0-preview-1220'; 3,352 votes; published 95% interval 1207.8–1230.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-1v-32k",
    "bench": "lmarena-vision",
    "score": 1064.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'step-1v-32k'; 1,534 votes; published 95% interval 1047.4–1080.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "seed-2-1-pro-preview",
    "bench": "lmarena-webdev",
    "score": 1516.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'seed-2.1-pro-preview'; 13,312 votes; published 95% interval 1510.7–1523.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kat-coder-pro-v1",
    "bench": "lmarena-webdev",
    "score": 1255.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'KAT-Coder-Pro-V1'; 1,504 votes; published 95% interval 1235.8–1274.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "longcat-flash-chat-2602-exp",
    "bench": "lmarena-text",
    "score": 1436.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'longcat-flash-chat-2602-exp'; 29,211 votes; published 95% interval 1432.1–1441.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-pro",
    "bench": "lmarena-text",
    "score": 1447.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mimo-v2-pro'; 25,396 votes; published 95% interval 1443.3–1452.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-pro",
    "bench": "lmarena-webdev",
    "score": 1432.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mimo-v2-pro'; 7,253 votes; published 95% interval 1424.6–1440.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reka-core-20240904",
    "bench": "lmarena-text",
    "score": 1288,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'reka-core-20240904'; 7,312 votes; published 95% interval 1280.8–1295.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reka-flash-20240904",
    "bench": "lmarena-text",
    "score": 1272.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'reka-flash-20240904'; 7,536 votes; published 95% interval 1265.4–1279.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "lmarena-text",
    "score": 1226.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'reka-flash-21b-20240226'; 24,806 votes; published 95% interval 1220.8–1232.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "yi-lightning",
    "bench": "lmarena-text",
    "score": 1328.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'yi-lightning'; 27,332 votes; published 95% interval 1323.3–1333.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "yi-vision",
    "bench": "lmarena-vision",
    "score": 1002.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'yi-vision'; 1,219 votes; published 95% interval 984.2–1020.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sonar-pro",
    "bench": "lmarena-search",
    "score": 1142.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'ppl-sonar-pro-high'; 28,519 votes; published 95% interval 1137.1–1148.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sonar-reasoning-pro",
    "bench": "lmarena-search",
    "score": 1138.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'ppl-sonar-reasoning-pro-high'; 29,055 votes; published 95% interval 1132.2–1144.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "p-image",
    "bench": "lmarena-image",
    "score": 1033.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'p-image'; 109,762 votes; published 95% interval 1029.8–1036.7; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1033.1), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "p-image-edit",
    "bench": "lmarena-image-edit",
    "score": 1210.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_edit/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-30",
    "note": "image_edit, category overall, split latest; exact entry 'p-image-edit'; 359,965 votes; published 95% interval 1207.6–1213.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "lmarena-text",
    "score": 1413.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-vl-235b-a22b-instruct'; 11,241 votes; published 95% interval 1406.9–1420.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "lmarena-vision",
    "score": 1215.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3-vl-235b-a22b-instruct'; 12,728 votes; published 95% interval 1208.1–1222.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "lmarena-text",
    "score": 1395,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'qwen3-vl-235b-a22b-thinking'; 7,805 votes; published 95% interval 1388.2–1401.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "lmarena-vision",
    "score": 1188,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'qwen3-vl-235b-a22b-thinking'; 2,388 votes; published 95% interval 1175.6–1200.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-5v",
    "bench": "lmarena-text",
    "score": 1352.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.5v'; 4,807 votes; published 95% interval 1343.9–1360.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-5v",
    "bench": "lmarena-vision",
    "score": 1152.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'glm-4.5v'; 3,249 votes; published 95% interval 1140.9–1164.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-6v",
    "bench": "lmarena-text",
    "score": 1378.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4.6v'; 2,830 votes; published 95% interval 1367.5–1389.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-6v",
    "bench": "lmarena-vision",
    "score": 1161.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'glm-4.6v'; 2,455 votes; published 95% interval 1147.7–1175.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "molmo-2-8b",
    "bench": "lmarena-text",
    "score": 1321.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'molmo-2-8b'; 818 votes; published 95% interval 1300.6–1343.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "molmo-2-8b",
    "bench": "lmarena-vision",
    "score": 1111.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'molmo-2-8b'; 1,192 votes; published 95% interval 1092.0–1131.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "lmarena-text",
    "score": 1306.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'olmo-3-32b-think'; 5,811 votes; published 95% interval 1298.1–1314.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "olmo-3-1-32b-instruct",
    "bench": "lmarena-text",
    "score": 1329.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'olmo-3.1-32b-instruct'; 11,801 votes; published 95% interval 1323.0–1335.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "intellect-3",
    "bench": "lmarena-text",
    "score": 1355.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'intellect-3'; 5,417 votes; published 95% interval 1347.5–1364.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ling-flash-2-0",
    "bench": "lmarena-text",
    "score": 1343.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ling-flash-2.0'; 6,887 votes; published 95% interval 1336.5–1351.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ring-flash-2-0",
    "bench": "lmarena-text",
    "score": 1322.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'ring-flash-2.0'; 6,932 votes; published 95% interval 1315.3–1329.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-3",
    "bench": "lmarena-text",
    "score": 1349.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'step-3'; 6,266 votes; published 95% interval 1341.9–1356.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-3",
    "bench": "lmarena-vision",
    "score": 1144.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'step-3'; 3,196 votes; published 95% interval 1133.0–1156.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "kimi-k2-0905",
    "bench": "lmarena-text",
    "score": 1418.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'kimi-k2-0905-preview'; 11,531 votes; published 95% interval 1412.2–1425.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "lmarena-text",
    "score": 1184.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-llm-67b-chat'; 4,932 votes; published 95% interval 1173.4–1196.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "lmarena-text",
    "score": 1265.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-coder-v2'; 15,147 votes; published 95% interval 1259.0–1271.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v2-5",
    "bench": "lmarena-text",
    "score": 1307.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v2.5'; 24,572 votes; published 95% interval 1302.5–1311.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "lmarena-text",
    "score": 1323.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v2.5-1210'; 6,795 votes; published 95% interval 1315.1–1331.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "lmarena-text",
    "score": 1417.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3.1-terminus-thinking'; 3,354 votes; published 95% interval 1407.1–1427.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "lmarena-text",
    "score": 1424.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v3.2-exp-thinking'; 8,947 votes; published 95% interval 1418.4–1431.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "lmarena-webdev",
    "score": 1271.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v3.2-exp'; 3,972 votes; published 95% interval 1258.3–1285.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "lmarena-text",
    "score": 1455,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v4-pro-high-preview' (the renamed '-thinking' entry, highest reasoning effort; corrected 2026-10-10 from the non-thinking 'deepseek-v4-pro' entry); 54,767 votes; published 95% interval 1451.0–1458.9. Pinned October dataset revision 46919c4."
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "lmarena-webdev",
    "score": 1463.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v4-pro-high-preview' (the renamed '-thinking' entry, highest reasoning effort; corrected 2026-10-10 from the non-thinking 'deepseek-v4-pro' entry); 13,229 votes; published 95% interval 1457.1–1469.7. Pinned October dataset revision 46919c4."
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "lmarena-text",
    "score": 1438.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'deepseek-v4-flash-high-preview' (the renamed '-thinking' entry, highest reasoning effort; corrected 2026-10-10 from the non-thinking 'deepseek-v4-flash' entry); 51,648 votes; published 95% interval 1434.3–1442.5. Pinned October dataset revision 46919c4."
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "lmarena-webdev",
    "score": 1430.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v4-flash-high-preview'; 10,158 votes; published 95% interval 1423.3–1437.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "devstral-2",
    "bench": "lmarena-webdev",
    "score": 1196.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'devstral-2'; 1,182 votes; published 95% interval 1175.4–1216.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "jamba-1-5-large",
    "bench": "lmarena-text",
    "score": 1289.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'jamba-1.5-large'; 8,662 votes; published 95% interval 1282.1–1296.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "lmarena-text",
    "score": 1239.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'jamba-1.5-mini'; 8,858 votes; published 95% interval 1232.5–1247.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "trinity-large-preview",
    "bench": "lmarena-text",
    "score": 1378.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'trinity-large-preview'; 31,201 votes; published 95% interval 1374.3–1382.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "trinity-large-thinking",
    "bench": "lmarena-text",
    "score": 1367.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'trinity-large-thinking'; 30,265 votes; published 95% interval 1362.7–1371.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "trinity-large-thinking",
    "bench": "lmarena-webdev",
    "score": 1237.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'trinity-large-thinking'; 1,362 votes; published 95% interval 1217.6–1257.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hy3-preview",
    "bench": "lmarena-text",
    "score": 1408.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-hy3-preview'; 6,874 votes; published 95% interval 1401.4–1416.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hy3-preview",
    "bench": "lmarena-webdev",
    "score": 1355.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'hunyuan-hy3-preview'; 1,455 votes; published 95% interval 1338.5–1373.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hy4-preview",
    "bench": "lmarena-webdev",
    "score": 1633.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'hy4-preview'; 5,088 votes; published 95% interval 1623.8–1642.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "lmarena-webdev",
    "score": 1618.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mimo-v2.6-pro'; 2,734 votes; published 95% interval 1606.2–1630.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "lmarena-text",
    "score": 1485.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3-pro'; 41,910 votes; published 95% interval 1481.7–1489.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "lmarena-vision",
    "score": 1289.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3-pro'; 13,620 votes; published 95% interval 1281.9–1296.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "lmarena-webdev",
    "score": 1439,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3-pro'; 14,092 votes; published 95% interval 1430.6–1447.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "lmarena-document",
    "score": 1451.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-3-pro'; 10,769 votes; published 95% interval 1442.1–1460.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "lmarena-search",
    "score": 1201.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gemini-3-pro-grounding'; 37,024 votes; published 95% interval 1195.8–1206.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "lmarena-text",
    "score": 1472.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-3-flash'; 31,259 votes; published 95% interval 1468.4–1477.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "lmarena-vision",
    "score": 1272.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gemini-3-flash'; 39,142 votes; published 95% interval 1267.7–1278.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "lmarena-webdev",
    "score": 1438.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-3-flash'; 10,896 votes; published 95% interval 1430.3–1447.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "lmarena-document",
    "score": 1439.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/document_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-13",
    "note": "document_style_control, category overall, split latest; exact entry 'gemini-3-flash'; 7,158 votes; published 95% interval 1429.8–1448.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "lmarena-search",
    "score": 1191.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'gemini-3-flash-grounding'; 149,334 votes; published 95% interval 1186.5–1196.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "lmarena-text",
    "score": 1430.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'grok-4-1-fast-reasoning'; 57,427 votes; published 95% interval 1427.1–1433.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "lmarena-vision",
    "score": 1196.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'grok-4-1-fast-reasoning'; 13,385 votes; published 95% interval 1189.3–1203.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "lmarena-webdev",
    "score": 1241.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'grok-4-1-fast-reasoning'; 5,491 votes; published 95% interval 1230.3–1252.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "lmarena-search",
    "score": 1194.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/search_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest; exact entry 'grok-4-1-fast-search'; 81,507 votes; published 95% interval 1189.0–1199.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "lmarena-webdev",
    "score": 1581,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'deepseek-v4-flash-high'; 5,040 votes; published 95% interval 1571.7–1590.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "yi-34b-chat",
    "bench": "lmarena-text",
    "score": 1183.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'yi-34b-chat'; 15,483 votes; published 95% interval 1176.9–1190.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "lmarena-text",
    "score": 1212.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'yi-1.5-34b-chat'; 24,146 votes; published 95% interval 1207.8–1217.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hailuo-02",
    "bench": "lmarena-video",
    "score": 1181.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'hailuo-02-standard'; 9,379 votes; published 95% interval 1169.0–1193.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hailuo-02",
    "bench": "lmarena-video-i2v",
    "score": 1223.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'hailuo-02-standard'; 21,785 votes; published 95% interval 1214.2–1233.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "sdxl-0-9",
    "bench": "wise-wiscore-overall",
    "score": 0.43,
    "source": "https://arxiv.org/html/2503.07265v4#S4.T1",
    "source_id": "visual-wise",
    "by": "independent",
    "date": "2026-06-02",
    "note": "SD-XL-base-0.9; Table 1 (S4.T1); Overall 0.43. WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge. Checkpoint stable-diffusion-xl-base-0.9 (Sec. 4.1 model list). Adapted row licensed CC-BY-SA-4.0 in visual-lanes2-cc-by-sa.json.",
    "evidence": {
      "source_model_label": "SD-XL-base-0.9",
      "table": "S4.T1",
      "reported_value": "0.43",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "WISE v4 Table 1 Overall WiScore; 1,000 original prompts; official default configuration per model with fixed random seed; GPT-4o-2024-05-13 judge.",
      "source_row": [
        "SD-XL-base-0.9",
        "0.43",
        "0.48",
        "0.47",
        "0.44",
        "0.45",
        "0.27",
        "0.43"
      ],
      "date_basis": "arXiv v4 submission date (2 Jun 2026), not a generation date.",
      "model_identity_scope": "Exact checkpoint named in Sec. 4.1 (\"stable-diffusion-xl-base-0.9 (Podell et al., 2023)\"; \"we use SD to denote stable-diffusion\"); no seed value, resolution or sampler beyond 'official default configurations' claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "Sec. 4.1: \"We evaluate 20 T2I models, including 10 dedicated T2I models and 10 unified multimodal models ... For image generation, we use the official default configurations of each model and fix the random seed to ensure reproducibility.\"",
      "evaluator_relationship": "benchmark authors (Peking University Shenzhen Graduate School et al.) evaluating other providers' models"
    },
    "evaluator": "WISE authors (Peking University Shenzhen Graduate School et al.)",
    "source_group": "visual-wise:wise-wiscore-overall",
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/visual-lanes2-cc-by-sa.json"
  },
  {
    "model": "nano-banana-2",
    "bench": "qwen-image-bench-v1",
    "score": 0.5982,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Nano Banana 2.0; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier. Sec. 4.2 cites Google (2026) \"Nano banana 2\" (https://deepmind.google/models/gemini-image/flash/, accessed 2026-05-14) = Gemini 3.1 Flash Image.",
    "evidence": {
      "source_model_label": "Nano Banana 2.0",
      "table": "S4.T2",
      "reported_value": "59.82",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Nano Banana 2.0",
        "54.77",
        "61.08",
        "62.40",
        "54.28",
        "67.05",
        "59.82"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed. Sec. 4.2 cites Google (2026) \"Nano banana 2\" (https://deepmind.google/models/gemini-image/flash/, accessed 2026-05-14) = Gemini 3.1 Flash Image.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "qwen-image-bench-v1",
    "score": 0.5722,
    "source": "https://arxiv.org/html/2605.28091v1#S4.T2",
    "source_id": "visual-qwenbench",
    "by": "independent",
    "date": "2026-05-27",
    "note": "Seedream 5.0; S4.T2; Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier. Sec. 4.2 cites ByteDance Seed (2026b) \"Seedream 5.0 lite\" (https://seed.bytedance.com/en/seedream5_0_lite, accessed 2026-05-14); Seedream 5.0 Pro was not yet released at v1 (27 May 2026).",
    "evidence": {
      "source_model_label": "Seedream 5.0",
      "table": "S4.T2",
      "reported_value": "57.22",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Original Q-Judger overall; 1,000 stratified prompts; 56 rubric facets. Reported release label retained; no inference of API snapshot or image-quality tier.",
      "source_row": [
        "Seedream 5.0",
        "52.55",
        "58.40",
        "58.90",
        "51.92",
        "65.29",
        "57.22"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed. Sec. 4.2 cites ByteDance Seed (2026b) \"Seedream 5.0 lite\" (https://seed.bytedance.com/en/seedream5_0_lite, accessed 2026-05-14); Seedream 5.0 Pro was not yet released at v1 (27 May 2026).",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.28091v1#S4.SS2.p2",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model",
      "independence_limitation": "Same Alibaba/Qwen benchmark and judge for all rows. Other-provider rows are not own-model lab reports; this is not a neutral multi-lab consensus."
    },
    "evaluator": "Qwen-Image-Bench authors / Alibaba",
    "source_group": "visual-qwenbench:qwen-image-bench-v1"
  },
  {
    "model": "uniworld-v1",
    "bench": "unicbench-en-overall",
    "score": 0.56013,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UniWorld-v1 [22]; S6.T4 Overall-EN 5.6013 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 5.6013/10. Open-source model; settings not tabulated. [22] = Lin et al., UniWorld (arXiv 2506.03147).",
    "evidence": {
      "source_model_label": "UniWorld-v1 [22]",
      "table": "S6.T4",
      "reported_value": "5.6013",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "UniWorld-v1 [22]",
        "5.3055",
        "7.3091",
        "6.4827",
        "4.0160",
        "5.6013",
        "-",
        "-",
        "-",
        "-",
        "-"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "instructpix2pix",
    "bench": "unicbench-en-overall",
    "score": 0.29221,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "source_id": "visual-unicbench",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Instruct-Pix2Pix [4]; S6.T4 Overall-EN 2.9221 on the 0–10 gpt-4.1-2025-04-14 judge scale, stored as 2.9221/10. Open-source model; settings not tabulated. [4] = Brooks, Holynski, Efros, InstructPix2Pix (CVPR 2023); only one public checkpoint (timbrooks/instruct-pix2pix).",
    "evidence": {
      "source_model_label": "Instruct-Pix2Pix [4]",
      "table": "S6.T4",
      "reported_value": "2.9221",
      "divisor": 10,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "UnicBench EN; per-sample geometric mean of applicable IF/NC/VQ/RA 0–10 scores; gpt-4.1-2025-04-14 judge.",
      "source_row": [
        "Instruct-Pix2Pix [4]",
        "2.8526",
        "4.0983",
        "3.9672",
        "1.9560",
        "2.9221",
        "-",
        "-",
        "-",
        "-",
        "-"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2512.02790v1#S6.SS1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "UnicEdit-10M authors (Zhejiang University / Tencent WeChat Vision)",
    "source_group": "visual-unicbench:unicbench-en-overall"
  },
  {
    "model": "instructpix2pix",
    "bench": "imagenworld-tie-human",
    "score": 0.45,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "source_id": "visual-imagenworld",
    "by": "independent",
    "date": "2026-03-29",
    "note": "InstructPix2Pix; Table 3 Text-guided Image Editing (TIE) human overall 0.45±0.22 (mean±sd across images); default configuration; InstructPix2Pix (Brooks et al., 2023b), single public checkpoint timbrooks/instruct-pix2pix.",
    "evidence": {
      "source_model_label": "InstructPix2Pix",
      "table": "S5.T3",
      "reported_value": "0.45",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Text-guided Image Editing (TIE); three human annotators per image, four 5-point Likert criteria rescaled to [0,1] and averaged.",
      "source_row": [
        "InstructPix2Pix",
        "0.26±\\pm0.24",
        "0.48±\\pm0.25",
        "0.56±\\pm0.26",
        "0.51±\\pm0.27",
        "0.45±\\pm0.22",
        "0.41±\\pm0.30",
        "0.64",
        "0.66"
      ],
      "source_header": [
        "Model",
        "Prompt Relevance",
        "Aesthetic Quality",
        "Content Coherence",
        "Artifacts",
        "O_Human",
        "O_VLM",
        "Krippendorff alpha",
        "Spearman rho"
      ],
      "task_section": "Text-guided Image Editing (TIE)",
      "date_basis": "Report version date (2026-03-29), not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/checkpoint (Table 2: InstructPix2Pix (Brooks et al., 2023b), Diffusion, TIE only); no undisclosed API snapshot or quality setting is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2603.27862v1#S4",
      "provenance_quote": "For fairness, all model implementations followed the default configurations recommended in their respective papers or official releases, and all open-source models were evaluated under the same seed (42). Approximately $1,000 USD were spent on API access for closed-source models and VLM-based evaluators.",
      "evaluator_relationship": "academic benchmark authors (University of Waterloo et al.) evaluating other providers' models"
    },
    "evaluator": "ImagenWorld authors (University of Waterloo et al.)",
    "source_group": "visual-imagenworld:imagenworld-tie-human"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "fepbench-if",
    "score": 0.575,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "Seedream 5.0; Table 2 IF mean 0.575 (RE 0.255, SP 0.716 not imported); API, 1920×1920/1920×1080/1080×1920; tested April 2026. Label cites [3] ByteDance Seed Team (2026) \"Seedream 5.0 Lite\" (https://seed.bytedance.com/en/seedream5_0_lite, accessed 2026-05-06).",
    "evidence": {
      "source_model_label": "Seedream 5.0",
      "table": "S4.T2",
      "reported_value": "0.575",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "Seedream 5.0",
        "0.748",
        "0.362",
        "0.505",
        "0.685",
        "0.266",
        "0.100",
        "0.398",
        "0.869",
        "0.485",
        "0.793",
        "0.575",
        "0.255",
        "0.716"
      ],
      "configuration_row": [
        "Seedream 5.0",
        "API",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "Not exposed",
        "Not exposed",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Tier from the cited reference [3] \"Seedream 5.0 Lite\"; API snapshot not stated.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "ovis-image-7b",
    "bench": "fepbench-if",
    "score": 0.334,
    "source": "https://arxiv.org/html/2606.05949v2#S4.T2",
    "source_id": "visual-fepbench",
    "by": "independent",
    "date": "2026-06-05",
    "note": "Ovis-Image; Table 2 IF mean 0.334 (RE 0.125, SP 0.618 not imported); open weights, 50 steps, guidance 5.0, 1920×1920/1920×1080/1080×1920; tested April 2026. Ref [26] Ovis-Image Technical Report (arXiv 2511.22982); only one public checkpoint (Ovis-Image-7B).",
    "evidence": {
      "source_model_label": "Ovis-Image",
      "table": "S4.T2",
      "reported_value": "0.334",
      "divisor": 1,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Identical prompts, both free-form and structured regimes; atom-level instruction faithfulness scored by MLLM evaluators; settings per Table 6.",
      "source_row": [
        "Ovis-Image",
        "0.478",
        "0.101",
        "0.173",
        "0.584",
        "0.174",
        "0.023",
        "0.179",
        "0.825",
        "0.230",
        "0.800",
        "0.334",
        "0.125",
        "0.618"
      ],
      "configuration_row": [
        "Ovis-Image",
        "Open-source",
        "1920×\\times1920 / 1920×\\times1080 / 1080×\\times1920",
        "50",
        "5.0",
        "None"
      ],
      "date_basis": "Report version date (v2, 2026-06-05); generation in April 2026 per Table 6",
      "model_identity_scope": "Exact source-reported release; single public checkpoint Ovis-Image-7B.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2606.05949v2#S5.SS0.SSS0.Px1.p1.1",
      "provenance_quote": "We benchmark nine text-to-image models ... All models are evaluated with identical prompts under the same benchmark split ... We do not use model-specific prompt engineering, manual image select[ion]",
      "evaluator_relationship": "academic benchmark authors (Shanghai AI Laboratory et al.) evaluating other providers' models"
    },
    "evaluator": "FEPBench authors (Shanghai AI Laboratory / Shanghai Innovation Institute et al.)",
    "source_group": "visual-fepbench:fepbench-if"
  },
  {
    "model": "uniworld-v1",
    "bench": "edit-compass-en-overall",
    "score": 0.342,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "UniWorld-V1; S4.T3 English-instruction Overall 1.71 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.71/5. Open model; default settings not further specified.",
    "evidence": {
      "source_model_label": "UniWorld-V1 Lin et al. (2025)",
      "table": "S4.T3",
      "reported_value": "1.71",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "UniWorld-V1 Lin et al. (2025)",
        "✓ \\checkmark",
        "2.94",
        "3.19",
        "3.09",
        "1.85",
        "3.34",
        "2.93",
        "1.47",
        "3.17",
        "3.27",
        "1.06",
        "1.59",
        "3.59",
        "1.74",
        "2.02",
        "2.81",
        "1.74",
        "2.65",
        "2.41",
        "1.71"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "instructpix2pix",
    "bench": "edit-compass-en-overall",
    "score": 0.238,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "InstructPix2Pix; S4.T3 English-instruction Overall 1.19 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.19/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "InstructPix2Pix Brooks et al. (2023)",
      "table": "S4.T3",
      "reported_value": "1.19",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "InstructPix2Pix Brooks et al. (2023)",
        "✗",
        "1.98",
        "1.34",
        "2.33",
        "1.72",
        "1.61",
        "2.23",
        "1.51",
        "1.56",
        "2.57",
        "1.09",
        "1.66",
        "3.26",
        "-",
        "-",
        "-",
        "1.91",
        "1.15",
        "2.07",
        "1.19"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "onecat-3b",
    "bench": "edit-compass-en-overall",
    "score": 0.242,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "OneCAT; S4.T3 English-instruction Overall 1.21 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.21/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input). OneCAT-3B is the only released OneCAT checkpoint.",
    "evidence": {
      "source_model_label": "OneCAT Li et al. (2025a)",
      "table": "S4.T3",
      "reported_value": "1.21",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "OneCAT Li et al. (2025a)",
        "✗",
        "2.23",
        "1.12",
        "2.12",
        "1.67",
        "1.26",
        "2.15",
        "1.41",
        "1.29",
        "2.09",
        "1.02",
        "1.05",
        "2.06",
        "-",
        "-",
        "-",
        "1.79",
        "1.03",
        "1.99",
        "1.21"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "lumina-dimoo",
    "bench": "edit-compass-en-overall",
    "score": 0.266,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Lumina-DiMOO; S4.T3 English-instruction Overall 1.33 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.33/5. Open model; default settings not further specified. No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "Lumina-DiMOO Xin et al. (2025)",
      "table": "S4.T3",
      "reported_value": "1.33",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "Lumina-DiMOO Xin et al. (2025)",
        "✗",
        "2.37",
        "2.00",
        "2.33",
        "1.73",
        "2.31",
        "2.40",
        "1.50",
        "2.89",
        "2.93",
        "1.09",
        "2.08",
        "2.83",
        "-",
        "-",
        "-",
        "1.70",
        "1.54",
        "2.08",
        "1.33"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "chronoedit-14b",
    "bench": "edit-compass-en-overall",
    "score": 0.37,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "source_id": "visual-edit-compass",
    "by": "independent",
    "date": "2026-05-13",
    "note": "ChronoEdit; S4.T3 English-instruction Overall 1.85 on the 1–5 Gemini-3.1-Pro judge scale, stored as 1.85/5. Open model; default settings not further specified (temporal-reasoning and distillation-LoRA options not stated). No Multi-Image cells (model lacks multi-image input).",
    "evidence": {
      "source_model_label": "ChronoEdit Wu et al. (2025c)",
      "table": "S4.T3",
      "reported_value": "1.85",
      "divisor": 5,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "protocol": "Edit-Compass EN, Overall = per-sample failure-gated weighted geometric mean of IA/VC/VQ; Gemini-3.1-Pro judge; 1–5 scale.",
      "source_row": [
        "ChronoEdit Wu et al. (2025c)",
        "✗",
        "2.63",
        "3.63",
        "3.49",
        "2.85",
        "3.84",
        "3.22",
        "1.61",
        "3.26",
        "3.35",
        "1.15",
        "1.87",
        "3.45",
        "-",
        "-",
        "-",
        "1.97",
        "3.00",
        "2.83",
        "1.85"
      ],
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release/tier; no undisclosed checkpoint hash or hosted API snapshot is claimed.",
      "original_measurement_verified": true,
      "provenance_evidence": "https://arxiv.org/html/2605.13062v1#A4.p1",
      "evaluator_relationship": "benchmark authors evaluating another provider’s model"
    },
    "evaluator": "Edit-Compass authors (HDU / PKU / Kling Team / CASIA)",
    "source_group": "visual-edit-compass:edit-compass-en-overall"
  },
  {
    "model": "sd-1-3",
    "bench": "geneval",
    "score": 0.43,
    "source": "https://arxiv.org/html/2310.11513v1#A1.T3",
    "source_id": "visual-geneval",
    "by": "independent",
    "date": "2023-10-17",
    "evaluator": "GenEval authors (Ghosh, Hajishirzi, Schmidt)",
    "source_group": "visual-geneval:geneval",
    "note": "SD v1.3; App. Table 3 (A1.T3), Overall column; reported 0.43; GenEval authors' run; same experiment as the approved README rows.",
    "evidence": {
      "reported_value": "0.43",
      "divisor": 1,
      "source_model_label": "SD v1.3",
      "table": "App. Table 3 (A1.T3), Overall column",
      "protocol": "Original 553-prompt GenEval, 4 images per prompt, model default resolution/steps/sampler (Sec. 5.1, App. C.2)",
      "measurement_origin": "benchmark/evaluator authors' own run",
      "date_basis": "Pinned source version date, not an asserted generation date",
      "original_measurement_verified": true,
      "source_row": [
        "SD v1.3",
        "0.43",
        "0.97",
        "0.38",
        "0.35",
        "0.77",
        "0.03",
        "0.05"
      ]
    }
  },
  {
    "model": "dall-e-3",
    "bench": "geckonum-task1-exact",
    "score": 0.452,
    "source": "https://arxiv.org/html/2406.14774v3#S5.T2",
    "source_id": "visual-geckonum",
    "source_group": "visual-geckonum:geckonum-task1-exact",
    "evaluator": "GeckoNum authors (Google DeepMind), human annotation",
    "by": "independent",
    "date": "2025-02-06",
    "note": "DALL·E 3; Table 2 (S5.T2) Task 1 Exact Number Generation 45.2 ± 0.5 (% accuracy, human-annotated; random chance 10.0); stored /100. Tasks 2-3 not imported. Identity: ref [2] Betker et al., Improving image generation with better captions (DALL·E 3).",
    "evidence": {
      "source_model_label": "DALL·E 3",
      "table": "Table 2 (S5.T2)",
      "reported_value": "45.2 ± 0.5",
      "divisor": 100,
      "measurement_origin": "authors generated five images per prompt with five seeds and collected five human annotations per image",
      "date_basis": "arXiv v3 date (6 Feb 2025). The DALL·E 3 row (45.2) is already in v1 (20 Jun 2024); the Stable Diffusion rows first appear in a later version, so v3 is the pinned revision.",
      "identity_evidence": "ref [2] Betker et al., Improving image generation with better captions (DALL·E 3)",
      "source_artifact_sha256": "4f081b732b8af5d65b567e8c13545b850dc667a7b0fe4e6cb122eff7ffbbc09c"
    }
  },
  {
    "model": "sd-1-5",
    "bench": "geckonum-task1-exact",
    "score": 0.203,
    "source": "https://arxiv.org/html/2406.14774v3#S5.T2",
    "source_id": "visual-geckonum",
    "source_group": "visual-geckonum:geckonum-task1-exact",
    "evaluator": "GeckoNum authors (Google DeepMind), human annotation",
    "by": "independent",
    "date": "2025-02-06",
    "note": "SD1.5; Table 2 (S5.T2) Task 1 Exact Number Generation 20.3 ± 0.4 (% accuracy, human-annotated; random chance 10.0); stored /100. Tasks 2-3 not imported. Identity: Stable Diffusion family refs [21,23,24]; label SD1.5 = Stable Diffusion v1.5 (WISE/TIFA precedent).",
    "evidence": {
      "source_model_label": "SD1.5",
      "table": "Table 2 (S5.T2)",
      "reported_value": "20.3 ± 0.4",
      "divisor": 100,
      "measurement_origin": "authors generated five images per prompt with five seeds and collected five human annotations per image",
      "date_basis": "arXiv v3 date (6 Feb 2025). The DALL·E 3 row (45.2) is already in v1 (20 Jun 2024); the Stable Diffusion rows first appear in a later version, so v3 is the pinned revision.",
      "identity_evidence": "Stable Diffusion family refs [21,23,24]; label SD1.5 = Stable Diffusion v1.5 (WISE/TIFA precedent)",
      "source_artifact_sha256": "4f081b732b8af5d65b567e8c13545b850dc667a7b0fe4e6cb122eff7ffbbc09c"
    }
  },
  {
    "model": "sd-2-1",
    "bench": "geckonum-task1-exact",
    "score": 0.258,
    "source": "https://arxiv.org/html/2406.14774v3#S5.T2",
    "source_id": "visual-geckonum",
    "source_group": "visual-geckonum:geckonum-task1-exact",
    "evaluator": "GeckoNum authors (Google DeepMind), human annotation",
    "by": "independent",
    "date": "2025-02-06",
    "note": "SD2.1; Table 2 (S5.T2) Task 1 Exact Number Generation 25.8 ± 0.4 (% accuracy, human-annotated; random chance 10.0); stored /100. Tasks 2-3 not imported. Identity: label SD2.1 = Stable Diffusion v2.1 (TIFA/GenEval/WISE precedent; 512-base vs 768 not stated).",
    "evidence": {
      "source_model_label": "SD2.1",
      "table": "Table 2 (S5.T2)",
      "reported_value": "25.8 ± 0.4",
      "divisor": 100,
      "measurement_origin": "authors generated five images per prompt with five seeds and collected five human annotations per image",
      "date_basis": "arXiv v3 date (6 Feb 2025). The DALL·E 3 row (45.2) is already in v1 (20 Jun 2024); the Stable Diffusion rows first appear in a later version, so v3 is the pinned revision.",
      "identity_evidence": "label SD2.1 = Stable Diffusion v2.1 (TIFA/GenEval/WISE precedent; 512-base vs 768 not stated)",
      "source_artifact_sha256": "4f081b732b8af5d65b567e8c13545b850dc667a7b0fe4e6cb122eff7ffbbc09c"
    }
  },
  {
    "model": "sdxl",
    "bench": "geckonum-task1-exact",
    "score": 0.228,
    "source": "https://arxiv.org/html/2406.14774v3#S5.T2",
    "source_id": "visual-geckonum",
    "source_group": "visual-geckonum:geckonum-task1-exact",
    "evaluator": "GeckoNum authors (Google DeepMind), human annotation",
    "by": "independent",
    "date": "2025-02-06",
    "note": "SDXL; Table 2 (S5.T2) Task 1 Exact Number Generation 22.8 ± 0.4 (% accuracy, human-annotated; random chance 10.0); stored /100. Tasks 2-3 not imported. Identity: ref [21] Podell et al. SDXL; SDXL 1.0 base (ImageTime/ImagenWorld precedent).",
    "evidence": {
      "source_model_label": "SDXL",
      "table": "Table 2 (S5.T2)",
      "reported_value": "22.8 ± 0.4",
      "divisor": 100,
      "measurement_origin": "authors generated five images per prompt with five seeds and collected five human annotations per image",
      "date_basis": "arXiv v3 date (6 Feb 2025). The DALL·E 3 row (45.2) is already in v1 (20 Jun 2024); the Stable Diffusion rows first appear in a later version, so v3 is the pinned revision.",
      "identity_evidence": "ref [21] Podell et al. SDXL; SDXL 1.0 base (ImageTime/ImagenWorld precedent)",
      "source_artifact_sha256": "4f081b732b8af5d65b567e8c13545b850dc667a7b0fe4e6cb122eff7ffbbc09c"
    }
  },
  {
    "model": "seedance-2-0",
    "bench": "worldreasonbench-human-elo",
    "score": 1471,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Seedance2.0; Table 4 (S4.T4) Human Elo 1471 (human rank 1 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). Seedance2.0 accessed through ByteDance's commercial API (Appendix R). Judge Elo (1183) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "Seedance2.0",
      "table": "Table 4 (S4.T4)",
      "reported_value": "1471",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "Seedance2.0 accessed through ByteDance's commercial API (Appendix R)",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "veo-3-1-fast",
    "bench": "worldreasonbench-human-elo",
    "score": 1253,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Veo3.1-Fast; Table 4 (S4.T4) Human Elo 1253 (human rank 2 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). Veo3.1-Fast accessed through Google's commercial API (Appendix R). Judge Elo (1151) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "Veo3.1-Fast",
      "table": "Table 4 (S4.T4)",
      "reported_value": "1253",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "Veo3.1-Fast accessed through Google's commercial API (Appendix R)",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "wan-2-6",
    "bench": "worldreasonbench-human-elo",
    "score": 1211,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Wan2.6; Table 4 (S4.T4) Human Elo 1211 (human rank 4 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). Wan2.6 accessed through Alibaba's commercial API (Appendix R). Judge Elo (1130) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "Wan2.6",
      "table": "Table 4 (S4.T4)",
      "reported_value": "1211",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "Wan2.6 accessed through Alibaba's commercial API (Appendix R)",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "sora-2",
    "bench": "worldreasonbench-human-elo",
    "score": 1118,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Sora2-8s; Table 4 (S4.T4) Human Elo 1118 (human rank 5 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). 'Sora2' accessed through OpenAI's commercial API (Appendix R); Sora2-8s row used (Sora2-12s, same model at 12 s, printed 1109, not imported). Judge Elo (1222) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "Sora2-8s",
      "table": "Table 4 (S4.T4)",
      "reported_value": "1118",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "'Sora2' accessed through OpenAI's commercial API (Appendix R); Sora2-8s row used (Sora2-12s, same model at 12 s, printed 1109, not imported)",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "worldreasonbench-human-elo",
    "score": 953,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Wan2.2-14B; Table 4 (S4.T4) Human Elo 953 (human rank 7 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). 'Wan2.2-14B' (Apache 2.0, official recipe) run from an image-plus-text input; the only 14B Wan2.2 checkpoint that takes an image is Wan2.2-I2V-A14B. Judge Elo (913) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "Wan2.2-14B",
      "table": "Table 4 (S4.T4)",
      "reported_value": "953",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "'Wan2.2-14B' (Apache 2.0, official recipe) run from an image-plus-text input; the only 14B Wan2.2 checkpoint that takes an image is Wan2.2-I2V-A14B",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "worldreasonbench-human-elo",
    "score": 911,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "HunyuanVideo-1.5; Table 4 (S4.T4) Human Elo 911 (human rank 8 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). HunyuanVideo-1.5 run from an image-plus-text input with the official recipe, i.e. the I2V checkpoint (VGI precedent). Judge Elo (841) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "HunyuanVideo-1.5",
      "table": "Table 4 (S4.T4)",
      "reported_value": "911",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "HunyuanVideo-1.5 run from an image-plus-text input with the official recipe, i.e. the I2V checkpoint (VGI precedent)",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "longcat-video",
    "bench": "worldreasonbench-human-elo",
    "score": 904,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "LongCat-Video; Table 4 (S4.T4) Human Elo 904 (human rank 9 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). LongCat-Video (Meituan LongCat License) official recipe; single checkpoint covers I2V. Judge Elo (876) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "LongCat-Video",
      "table": "Table 4 (S4.T4)",
      "reported_value": "904",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "LongCat-Video (Meituan LongCat License) official recipe; single checkpoint covers I2V",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "ltx-2-3",
    "bench": "worldreasonbench-human-elo",
    "score": 587,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "source_id": "video-worldreasonbench",
    "source_group": "video-worldreasonbench:worldreasonbench-human-elo",
    "evaluator": "WorldReasonBench authors (Tsinghua University, NTU, HKUST-GZ, 2077 AI, University of Waterloo)",
    "by": "independent",
    "date": "2026-05-11",
    "note": "LTX2.3; Table 4 (S4.T4) Human Elo 587 (human rank 11 of 11); expert Bradley-Terry/Davidson fit over ~6K WorldRewardBench preference pairs, 15 annotators; TI2V (initial image + instruction). LTX2.3 (Apache 2.0) official recipe; single checkpoint covers I2V. Judge Elo (802) and automatic QA columns not imported.",
    "evidence": {
      "source_model_label": "LTX2.3",
      "table": "Table 4 (S4.T4)",
      "reported_value": "587",
      "divisor": 1,
      "source_header": [
        "Human Rank",
        "Model",
        "Human Elo",
        "Judge Elo",
        "Judge Rank",
        "Acc_QA (%)",
        "|Δr|",
        "Score_PR (%)",
        "|Δr|",
        "s_dyn/Acc_QA"
      ],
      "measurement_origin": "benchmark authors' own expert human annotation, fitted to Elo by the authors",
      "date_basis": "arXiv v1 submission date (11 May 2026)",
      "identity_evidence": "LTX2.3 (Apache 2.0) official recipe; single checkpoint covers I2V",
      "source_artifact_sha256": "efb44409c085d7896c9d80fc55145d69f245b89b3ffcd877e049b9996a99a29d"
    }
  },
  {
    "model": "nova-0-6b-t2v",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.4118,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 41.18 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/baaivision/NOVA, cites NOVA paper [87] (arXiv 2412.14169), 768x480, 33 frames, 12 fps (no variable-setting dagger); NOVA's only text-to-video checkpoint is BAAI/nova-d48w1024-osp480 (NOVA-0.6B, 33x768x480, exported at fps=12 in the README quickstart).",
    "evidence": {
      "source_model_label": "NOVA [ 87 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "41.18",
      "divisor": 100,
      "source_row": [
        "NOVA [ 87 ]",
        "43.02",
        "47.73",
        "40.26",
        "43.05",
        "43.30",
        "36.30",
        "38.75",
        "33.44",
        "33.45",
        "36.50",
        "42.63",
        "39.84",
        "44.19",
        "48.40",
        "42.23",
        "43.37",
        "38.38",
        "40.16",
        "42.23",
        "42.92",
        "41.18",
        "23"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/baaivision/NOVA, cites NOVA paper [87] (arXiv 2412.14169), 768x480, 33 frames, 12 fps (no variable-setting dagger); NOVA's only text-to-video checkpoint is BAAI/nova-d48w1024-osp480 (NOVA-0.6B, 33x768x480, exported at fps=12 in the README quickstart)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "model_table_row": [
        "♡ \\heartsuit NOVA [ 87 ]",
        "33",
        "12",
        "768 × \\times 480",
        "https://github.com/baaivision/NOVA"
      ],
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "lvdm-t2v",
    "bench": "love-aigve60k-perception-mos",
    "score": 0.3384,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T8",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-perception-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human perception MOS, Overall column = 33.84 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 8 of arXiv 2505.12098v1. Identity: Table 7 github.com/YingqingHe/LVDM, cites the LVDM paper [91] (arXiv 2211.13221), 256x256, 16 frames, 8 fps (no dagger); LVDM's only text-to-video checkpoint is lvdm_short/t2v.ckpt (configs/lvdm_short/text2video.yaml: 32x32 latent = 256x256, video_length 16; sample_text2video.py save_fps 8).",
    "evidence": {
      "source_model_label": "LVDM [ 91 ]",
      "table": "Table 8 (A6.T8)",
      "reported_value": "33.84",
      "divisor": 100,
      "source_row": [
        "LVDM [ 91 ]",
        "37.95",
        "37.35",
        "32.14",
        "32.57",
        "29.72",
        "32.46",
        "28.48",
        "29.29",
        "32.44",
        "34.78",
        "33.78",
        "33.03",
        "34.25",
        "38.69",
        "31.92",
        "34.27",
        "33.43",
        "33.89",
        "32.76",
        "35.58",
        "33.84",
        "28"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/YingqingHe/LVDM, cites the LVDM paper [91] (arXiv 2211.13221), 256x256, 16 frames, 8 fps (no dagger); LVDM's only text-to-video checkpoint is lvdm_short/t2v.ckpt (configs/lvdm_short/text2video.yaml: 32x32 latent = 256x256, video_length 16; sample_text2video.py save_fps 8)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "model_table_row": [
        "♡ \\heartsuit LVDM [ 91 ]",
        "16",
        "8",
        "256 × \\times 256",
        "https://github.com/YingqingHe/LVDM"
      ],
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "nova-0-6b-t2v",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.4718,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 47.18 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/baaivision/NOVA, cites NOVA paper [87] (arXiv 2412.14169), 768x480, 33 frames, 12 fps (no variable-setting dagger); NOVA's only text-to-video checkpoint is BAAI/nova-d48w1024-osp480 (NOVA-0.6B, 33x768x480, exported at fps=12 in the README quickstart).",
    "evidence": {
      "source_model_label": "NOVA [ 87 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "47.18",
      "divisor": 100,
      "source_row": [
        "NOVA [ 87 ]",
        "52.23",
        "51.77",
        "48.03",
        "50.54",
        "40.35",
        "45.14",
        "45.10",
        "45.48",
        "43.94",
        "33.84",
        "50.91",
        "50.35",
        "46.09",
        "55.43",
        "53.20",
        "38.06",
        "46.33",
        "41.32",
        "37.32",
        "53.45",
        "47.18",
        "21"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/baaivision/NOVA, cites NOVA paper [87] (arXiv 2412.14169), 768x480, 33 frames, 12 fps (no variable-setting dagger); NOVA's only text-to-video checkpoint is BAAI/nova-d48w1024-osp480 (NOVA-0.6B, 33x768x480, exported at fps=12 in the README quickstart)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "model_table_row": [
        "♡ \\heartsuit NOVA [ 87 ]",
        "33",
        "12",
        "768 × \\times 480",
        "https://github.com/baaivision/NOVA"
      ],
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "lvdm-t2v",
    "bench": "love-aigve60k-correspondence-mos",
    "score": 0.422,
    "source": "https://arxiv.org/html/2505.12098v1#A6.T9",
    "source_id": "video-love-aigve60k",
    "source_group": "video-love-aigve60k:love-aigve60k-correspondence-mos",
    "evaluator": "LOVE / AIGVE-60K authors (Shanghai Jiao Tong University)",
    "by": "independent",
    "date": "2025-05-17",
    "note": "AIGVE-60K human text-video correspondence MOS, Overall column = 42.20 (0-100 scale: per-subject z-scores rescaled by 100(z+3)/6, averaged; 3,050 prompts x 30 models, 15 annotators per video). Table 9 of arXiv 2505.12098v1. Identity: Table 7 github.com/YingqingHe/LVDM, cites the LVDM paper [91] (arXiv 2211.13221), 256x256, 16 frames, 8 fps (no dagger); LVDM's only text-to-video checkpoint is lvdm_short/t2v.ckpt (configs/lvdm_short/text2video.yaml: 32x32 latent = 256x256, video_length 16; sample_text2video.py save_fps 8).",
    "evidence": {
      "source_model_label": "LVDM [ 91 ]",
      "table": "Table 9 (A6.T9)",
      "reported_value": "42.20",
      "divisor": 100,
      "source_row": [
        "LVDM [ 91 ]",
        "49.90",
        "43.44",
        "40.96",
        "43.66",
        "32.91",
        "44.49",
        "33.45",
        "42.63",
        "39.83",
        "34.30",
        "41.65",
        "44.66",
        "42.43",
        "47.83",
        "43.24",
        "38.82",
        "44.22",
        "38.41",
        "35.08",
        "46.48",
        "42.20",
        "27"
      ],
      "source_header": [
        "Models",
        "Object",
        "Color",
        "Count.",
        "Texture",
        "Position",
        "HOI",
        "Face",
        "Emo.",
        "Human",
        "OCR",
        "Scene",
        "Style",
        "Shape",
        "View",
        "Know.",
        "Ling.",
        "Imag.",
        "Motion",
        "Event",
        "Complex",
        "Overall",
        "Rank"
      ],
      "measurement_origin": "authors' own generation of 58,500 videos from 30 T2V models and own subjective study (ITU-R BT.500 outlier rejection)",
      "identity_evidence": "Table 7 github.com/YingqingHe/LVDM, cites the LVDM paper [91] (arXiv 2211.13221), 256x256, 16 frames, 8 fps (no dagger); LVDM's only text-to-video checkpoint is lvdm_short/t2v.ckpt (configs/lvdm_short/text2video.yaml: 32x32 latent = 256x256, video_length 16; sample_text2video.py save_fps 8)",
      "model_table": "https://arxiv.org/html/2505.12098v1#A4.T7",
      "model_table_row": [
        "♡ \\heartsuit LVDM [ 91 ]",
        "16",
        "8",
        "256 × \\times 256",
        "https://github.com/YingqingHe/LVDM"
      ],
      "date_basis": "arXiv v1 publication date; generation/access dates 2025-03-08 per references",
      "source_artifact_sha256": "6930413d181be9fa6ea07a2ad91d63fe3e34ea8db05056d2e82eb299140d8a91"
    }
  },
  {
    "model": "sora-2",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.207,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:sora-2-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Sora 2; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 8 s / 720p and VGGT world decoder; published WO 0.207. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Sora 2",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.207",
      "divisor": 1,
      "source_row": [
        "Sora 2",
        "✗",
        "8 - 720p",
        "1.912",
        "1.289",
        "54.84",
        "0.160",
        "0.788",
        "0.207"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "veo-3-1",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.21,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:veo-3-1-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Veo 3.1; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 8 s / 720p and VGGT world decoder; published WO 0.210. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Veo 3.1",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.210",
      "divisor": 1,
      "source_row": [
        "Veo 3.1",
        "✗",
        "8 - 720p",
        "3.125",
        "2.432",
        "77.25",
        "0.212",
        "0.450",
        "0.210"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "veo-3-1-fast",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.192,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:veo-3-1-fast-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Veo 3.1-fast; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 8 s / 720p and VGGT world decoder; published WO 0.192. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Veo 3.1-fast",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.192",
      "divisor": 1,
      "source_row": [
        "Veo 3.1-fast",
        "✗",
        "8 - 720p",
        "2.863",
        "1.798",
        "57.42",
        "0.140",
        "0.691",
        "0.192"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-5-preview",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.215,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan-2-5-preview-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.5-Preview † (Wan2.5-I2V-Preview in Sec. 4 and Table 4); Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 10 s / 720p and VGGT world decoder; published WO 0.215. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.5-Preview †",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.215",
      "divisor": 1,
      "source_row": [
        "Wan2.5-Preview †",
        "✗",
        "10 - 720p",
        "2.478",
        "1.596",
        "61.69",
        "0.182",
        "0.873",
        "0.215"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-2-i2v-plus",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.29,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan-2-2-i2v-plus-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.2-Plus † (Wan2.2-I2V-Plus in Table 4); Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 5 s / 1080p and VGGT world decoder; published WO 0.290. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.2-Plus †",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.290",
      "divisor": 1,
      "source_row": [
        "Wan2.2-Plus †",
        "✗",
        "5 - 1080p",
        "1.377",
        "1.044",
        "45.25",
        "0.240",
        "0.686",
        "0.290"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-2-i2v-flash",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.341,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan-2-2-i2v-flash-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.2-Flash † (Wan2.2-I2V-Flash in Table 4); Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 5 s / 720p and VGGT world decoder; published WO 0.341. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.2-Flash †",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.341",
      "divisor": 1,
      "source_row": [
        "Wan2.2-Flash †",
        "✗",
        "5 - 720p",
        "1.362",
        "1.005",
        "38.75",
        "0.292",
        "0.746",
        "0.341"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-1-i2v-plus",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.157,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan-2-1-i2v-plus-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.1-Plus † (I2V model per the Table 3 dagger); Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 5 s / 720p and VGGT world decoder; published WO 0.157. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.1-Plus †",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.157",
      "divisor": 1,
      "source_row": [
        "Wan2.1-Plus †",
        "✗",
        "5 - 720p",
        "2.782",
        "1.594",
        "53.90",
        "0.090",
        "0.715",
        "0.157"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-1-i2v-turbo",
    "bench": "target-bench-v2-vggt-wo-i2v",
    "score": 0.059,
    "source": "https://arxiv.org/html/2511.17792v2#S4.T3",
    "source_id": "visual-gap-target-bench",
    "source_group": "visual-gap-target-bench:target-bench-v2-vggt-wo-i2v",
    "evaluation_run_group": "visual-gap-target-bench:wan-2-1-i2v-turbo-vggt",
    "evaluator": "Target-Bench authors (TUM / Bosch AI Center / ETH / NJU)",
    "by": "independent",
    "date": "2026-04-15",
    "note": "Wan2.1-Turbo † (I2V model per the Table 3 dagger); FDE 4.243, SE 0.000, AC 1.000 as printed; Target-Bench authors’ I2V planning evaluation on 125 held-out scenarios, input first frame plus semantic target text; closed model accessed through its official API (Sec. 4.1). Table 3 reports 5 s / 720p and VGGT world decoder; published WO 0.059. Frame count, exact pixel dimensions, FPS, seed and prompt-extension setting not stated for this row. Use v2 scoring, not older v1 or SpaTracker/ViPE results.",
    "data_license": "CC-BY-4.0",
    "evidence": {
      "source_model_label": "Wan2.1-Turbo †",
      "table": "Table 3 (VGGT)",
      "reported_value": "0.059",
      "divisor": 1,
      "source_row": [
        "Wan2.1-Turbo †",
        "✗",
        "5 - 720p",
        "4.243",
        "1.850",
        "64.21",
        "0.000",
        "1.000",
        "0.059"
      ],
      "measurement_origin": "explicit independent evaluation by the benchmark authors (closed model via its official API)",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated model's author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/API tier as reported; no checkpoint revision or later model version is inferred.",
      "source_artifact_sha256": "e29453328df2b07283bc5785608d14d47d59add866badc2c30375c299cd74488",
      "task": "image-to-video"
    }
  },
  {
    "model": "wan-2-1-1-3b",
    "bench": "vbench-t2v-total",
    "score": 0.8328,
    "source": "https://arxiv.org/pdf/2505.16864v2#page=8",
    "source_id": "visual-gap-jenga",
    "source_group": "visual-gap-jenga:vbench-t2v-total",
    "evaluation_run_group": "visual-gap-jenga:wan21-1-3b-standard",
    "evaluator": "Jenga authors (CUHK / HKUST / Kuaishou / SmartMore)",
    "by": "independent",
    "date": "2025-11-22",
    "note": "Original Wan2.1 T2V 1.3B baseline rerun by Jenga authors: Table 2(b) row 'Wan2.1-1.3B [13]', 50 NFE, VBench 83.28% (same value in appendix Tables 4 and 6); latent size RS=[20, 30, 52] per Table 4; original full-set VBench prompts, fixed seed per prompt (seed value undisclosed). No Jenga/TeaCache acceleration rows; CFG, precision, FPS and checkpoint revision not reported.",
    "data_license": "CC-BY-SA-4.0",
    "evidence": {
      "source_model_label": "Wan2.1-1.3B [13]",
      "table": "Table 2(b), PDF page 8",
      "reported_value": "83.28",
      "divisor": 100,
      "measurement_origin": "explicit independent rerun / original evaluator measurement",
      "original_measurement_verified": true,
      "evaluator_relationship": "evaluator is not the evaluated checkpoint's model author",
      "date_basis": "Source publication/version date, not an asserted generation date.",
      "model_identity_scope": "Exact named release/checkpoint family and baseline variant as reported; source does not pin a checkpoint revision/hash.",
      "source_artifact_sha256": "81e5fc73bd50f88bba235bea55358e82e9b8e1ebf9e1d24710aa3b0c672d33e3",
      "task": "text-to-video",
      "steps": 50,
      "prompt_format": "standard/original"
    },
    "licensed_artifact": "/data/visual-gap-cc-by-sa.json"
  },
  {
    "model": "minimax-h3",
    "bench": "vgi-bench-i2v",
    "score": 0.444,
    "source": "https://arxiv.org/html/2608.19583v3#S3.T2",
    "source_id": "visual-vgi",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Mnx-H3; S3.T2; I2V problem-solving; published Overall final score, not success rate. Open-weights MiniMax-H3 run by the authors at 960×544, 24fps, 5/10s (Table 10 'Open Source' block; no per-second price) Fixed half-instance sample; preserve source v3. Table 2 places Mnx-H3 under the 'Open Source' header (colspan 3 with HY1.5 and Wan2.2), consistent with Table 10, so the commercial/open conflict that quarantined this row does not occur in the pinned v3 HTML.",
    "evidence": {
      "source_model_label": "Mnx-H3",
      "table": "S3.T2",
      "reported_value": "44.4",
      "divisor": 100,
      "measurement_origin": "explicit original benchmark-author evaluation",
      "source_row": [
        "Overall",
        "51.0",
        "36.7",
        "32.0",
        "44.0",
        "35.7",
        "36.6",
        "44.4",
        "19.1",
        "21.6"
      ],
      "configuration_table": "A2.T10",
      "date_basis": "Report publication/version date, not an asserted generation date",
      "model_identity_scope": "Exact source-reported release; open-weights MiniMax-H3 per Table 2 'Open Source' header (colspan 3) and Table 10.",
      "original_measurement_verified": true,
      "evaluator_relationship": "benchmark authors evaluating another provider's model",
      "source_artifact_sha256": "da2e1dbde56c3a72357a2094e0bba099f73d3fcfdf2740040da20122fa25ae1e"
    },
    "evaluator": "VGI-Bench authors",
    "source_group": "visual-vgi:vgi-bench-i2v"
  },
  {
    "model": "gpt-4-0613",
    "bench": "eq-bench-v1",
    "score": 0.6252,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI gpt-4-0613', column 'EQ-Bench Score' = 62.52 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI gpt-4-0613",
      "column_label": "EQ-Bench Score",
      "raw_value": "62.52",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-0314",
    "bench": "eq-bench-v1",
    "score": 0.5339,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI gpt-4-0314', column 'EQ-Bench Score' = 53.39 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI gpt-4-0314",
      "column_label": "EQ-Bench Score",
      "raw_value": "53.39",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "eq-bench-v1",
    "score": 0.5156,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'meta-llama/Llama-2-70b-chat-hf', column 'EQ-Bench Score' = 51.56 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "meta-llama/Llama-2-70b-chat-hf",
      "column_label": "EQ-Bench Score",
      "raw_value": "51.56",
      "scale_divisor": 100
    }
  },
  {
    "model": "yi-34b-chat",
    "bench": "eq-bench-v1",
    "score": 0.5103,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row '01-ai/Yi-34B-Chat', column 'EQ-Bench Score' = 51.03 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "01-ai/Yi-34B-Chat",
      "column_label": "EQ-Bench Score",
      "raw_value": "51.03",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "eq-bench-v1",
    "score": 0.4917,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI gpt-3.5-0613', column 'EQ-Bench Score' = 49.17 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI gpt-3.5-0613",
      "column_label": "EQ-Bench Score",
      "raw_value": "49.17",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "eq-bench-v1",
    "score": 0.4761,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI gpt-3.5-turbo-0301', column 'EQ-Bench Score' = 47.61 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI gpt-3.5-turbo-0301",
      "column_label": "EQ-Bench Score",
      "raw_value": "47.61",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen-14b-chat",
    "bench": "eq-bench-v1",
    "score": 0.4376,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'Qwen/Qwen-14B-Chat', column 'EQ-Bench Score' = 43.76 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "Qwen/Qwen-14B-Chat",
      "column_label": "EQ-Bench Score",
      "raw_value": "43.76",
      "scale_divisor": 100
    }
  },
  {
    "model": "text-davinci-003",
    "bench": "eq-bench-v1",
    "score": 0.4373,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI text-davinci-003', column 'EQ-Bench Score' = 43.73 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI text-davinci-003",
      "column_label": "EQ-Bench Score",
      "raw_value": "43.73",
      "scale_divisor": 100
    }
  },
  {
    "model": "text-davinci-002",
    "bench": "eq-bench-v1",
    "score": 0.3944,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'OpenAI text-davinci-002', column 'EQ-Bench Score' = 39.44 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "OpenAI text-davinci-002",
      "column_label": "EQ-Bench Score",
      "raw_value": "39.44",
      "scale_divisor": 100
    }
  },
  {
    "model": "openchat-3-5",
    "bench": "eq-bench-v1",
    "score": 0.3708,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'openchat/openchat_3.5', column 'EQ-Bench Score' = 37.08 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "openchat/openchat_3.5",
      "column_label": "EQ-Bench Score",
      "raw_value": "37.08",
      "scale_divisor": 100
    }
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "eq-bench-v1",
    "score": 0.3652,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'lmsys/vicuna-33b-v1.3', column 'EQ-Bench Score' = 36.52 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "lmsys/vicuna-33b-v1.3",
      "column_label": "EQ-Bench Score",
      "raw_value": "36.52",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "eq-bench-v1",
    "score": 0.3302,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'meta-llama/Llama-2-13b-chat-hf', column 'EQ-Bench Score' = 33.02 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "meta-llama/Llama-2-13b-chat-hf",
      "column_label": "EQ-Bench Score",
      "raw_value": "33.02",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "eq-bench-v1",
    "score": 0.2543,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'meta-llama/Llama-2-7b-chat-hf', column 'EQ-Bench Score' = 25.43 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "meta-llama/Llama-2-7b-chat-hf",
      "column_label": "EQ-Bench Score",
      "raw_value": "25.43",
      "scale_divisor": 100
    }
  },
  {
    "model": "koala-13b",
    "bench": "eq-bench-v1",
    "score": 0.2492,
    "source": "https://arxiv.org/html/2312.06281v2",
    "source_id": "paper-2312.06281v2",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "EQ-Bench paper (arXiv:2312.06281v2) Table 1 'EQ-Bench Score Comparison' (Section 4.1, the headline results table), row 'Koala 13B', column 'EQ-Bench Score' = 24.92 (divided by 100). EQ-Bench v1 (60 questions), zero-shot, temperature 0.01, critique-and-revision prompt, best of first-pass and revised scores; run by the benchmark author (Samuel J. Paech) in late 2023. Open weights run by the author with bitsandbytes quantisation (8-bit for 7B-34B, 4-bit for larger models; Section 3.10). Table 1 is used consistently; the appendix correlation Tables 3-5 print different values for some rows (e.g. gpt-4-0314 60.73, gpt-3.5-turbo-0301 49.52, and text-davinci-002/003 swapped); Table 6 repeat runs agree with Table 1.",
    "evidence": {
      "kind": "table",
      "table": "Table 1",
      "row_label": "Koala 13B",
      "column_label": "EQ-Bench Score",
      "raw_value": "24.92",
      "scale_divisor": 100
    }
  },
  {
    "model": "o1-mini",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.92,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'o1-mini-2024-09-12' score 92.0 (95% CI -1.2, 1.0; average #tokens 1399); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Reasoning model at its API default reasoning setting (the README states no effort level).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "o1-mini-2024-09-12",
      "column_label": "score",
      "raw_value": "92.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "o1-preview",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.904,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'o1-preview-2024-09-12' score 90.4 (95% CI -1.1, 1.3; average #tokens 1193); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Reasoning model at its API default reasoning setting (the README states no effort level).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "o1-preview-2024-09-12",
      "column_label": "score",
      "raw_value": "90.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.852,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-3-5-sonnet-20241022' score 85.2 (95% CI -1.4, 1.6; average #tokens 691); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-3-5-sonnet-20241022",
      "column_label": "score",
      "raw_value": "85.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "athene-v2-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.85,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'athene-v2-chat' score 85.0 (95% CI -1.4, 1.7; average #tokens 884); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "athene-v2-chat",
      "column_label": "score",
      "raw_value": "85.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.849,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3.1-nemotron-70b-instruct' score 84.9 (95% CI -1.7, 1.8; average #tokens 869); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3.1-nemotron-70b-instruct",
      "column_label": "score",
      "raw_value": "84.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.826,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4-turbo-2024-04-09' score 82.6 (95% CI -1.8, 1.5; average #tokens 662); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4-turbo-2024-04-09",
      "column_label": "score",
      "raw_value": "82.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "yi-lightning",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.815,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'yi-lightning' score 81.5 (95% CI -1.6, 1.6; average #tokens 875); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "yi-lightning",
      "column_label": "score",
      "raw_value": "81.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.793,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-3-5-sonnet-20240620' score 79.3 (95% CI -2.1, 2.0; average #tokens 567); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-3-5-sonnet-20240620",
      "column_label": "score",
      "raw_value": "79.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.792,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4o-2024-05-13' score 79.2 (95% CI -1.9, 1.7; average #tokens 696); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4o-2024-05-13",
      "column_label": "score",
      "raw_value": "79.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.78,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4-0125-preview' score 78.0 (95% CI -2.1, 2.4; average #tokens 619); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4-0125-preview",
      "column_label": "score",
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen2-5-72b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.78,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'qwen2.5-72b-instruct' score 78.0 (95% CI -1.8, 1.8; average #tokens 821); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "qwen2.5-72b-instruct",
      "column_label": "score",
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.779,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4o-2024-08-06' score 77.9 (95% CI -2.0, 2.1; average #tokens 594); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4o-2024-08-06",
      "column_label": "score",
      "raw_value": "77.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "athene-70b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.776,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'athene-70b' score 77.6 (95% CI -2.7, 2.2; average #tokens 684); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'athene-70b' (added 2024-07-25).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "athene-70b",
      "column_label": "score",
      "raw_value": "77.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.749,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4o-mini' score 74.9 (95% CI -2.5, 1.9; average #tokens 668); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'gpt-4o-mini' added 2024-07-25, when gpt-4o-mini-2024-07-18 was the only snapshot.",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4o-mini",
      "column_label": "score",
      "raw_value": "74.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-large-2407",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.704,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'mistral-large-2407' score 70.4 (95% CI -1.6, 2.1; average #tokens 623); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "mistral-large-2407",
      "column_label": "score",
      "raw_value": "70.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-405b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.693,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3.1-405b-instruct-fp8' score 69.3 (95% CI -2.4, 2.2; average #tokens 658); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Served in FP8 (label).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3.1-405b-instruct-fp8",
      "column_label": "score",
      "raw_value": "69.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "glm-4-0520",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.638,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'glm-4-0520' score 63.8 (95% CI -2.9, 2.8; average #tokens 636); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "glm-4-0520",
      "column_label": "score",
      "raw_value": "63.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.623,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'deepseek-coder-v2' score 62.3 (95% CI -2.1, 1.8; average #tokens 578); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'deepseek-coder-v2' added 2024-06-28 (same mapping as the Arena lane: DeepSeek-Coder-V2-Instruct 0614).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "deepseek-coder-v2",
      "column_label": "score",
      "raw_value": "62.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.604,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-3-opus-20240229' score 60.4 (95% CI -2.5, 2.5; average #tokens 541); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-3-opus-20240229",
      "column_label": "score",
      "raw_value": "60.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2-27b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.575,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gemma-2-27b-it' score 57.5 (95% CI -2.1, 2.4; average #tokens 577); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gemma-2-27b-it",
      "column_label": "score",
      "raw_value": "57.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-70b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.557,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3.1-70b-instruct' score 55.7 (95% CI -2.9, 2.7; average #tokens 628); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3.1-70b-instruct",
      "column_label": "score",
      "raw_value": "55.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen2-72b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.469,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'qwen2-72b-instruct' score 46.9 (95% CI -2.5, 2.7; average #tokens 515); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "qwen2-72b-instruct",
      "column_label": "score",
      "raw_value": "46.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.468,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-3-sonnet-20240229' score 46.8 (95% CI -2.3, 2.7; average #tokens 552); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-3-sonnet-20240229",
      "column_label": "score",
      "raw_value": "46.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-70b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.466,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3-70b-instruct' score 46.6 (95% CI -2.3, 2.6; average #tokens 591); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3-70b-instruct",
      "column_label": "score",
      "raw_value": "46.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.415,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-3-haiku-20240307' score 41.5 (95% CI -2.5, 2.5; average #tokens 505); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-3-haiku-20240307",
      "column_label": "score",
      "raw_value": "41.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-0613",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.379,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-4-0613' score 37.9 (95% CI -2.8, 2.4; average #tokens 354); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-4-0613",
      "column_label": "score",
      "raw_value": "37.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-large-2402",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.377,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'mistral-large-2402' score 37.7 (95% CI -2.1, 2.6; average #tokens 400); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "mistral-large-2402",
      "column_label": "score",
      "raw_value": "37.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.364,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'mixtral-8x22b-instruct-v0.1' score 36.4 (95% CI -2.4, 2.6; average #tokens 430); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "mixtral-8x22b-instruct-v0.1",
      "column_label": "score",
      "raw_value": "36.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.361,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Qwen1.5-72B-Chat' score 36.1 (95% CI -2.0, 2.7; average #tokens 474); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Qwen1.5-72B-Chat",
      "column_label": "score",
      "raw_value": "36.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.334,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'phi-3-medium-4k-instruct' score 33.4 (95% CI -2.6, 2.1; average #tokens 517); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "phi-3-medium-4k-instruct",
      "column_label": "score",
      "raw_value": "33.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.331,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'command-r-plus' score 33.1 (95% CI -2.8, 2.4; average #tokens 541); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'command-r-plus' added 2024-04-16, before the 08-2024 release, so the 04-2024 checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "command-r-plus",
      "column_label": "score",
      "raw_value": "33.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-medium-2312",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.319,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'mistral-medium' score 31.9 (95% CI -1.9, 2.2; average #tokens 485); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'mistral-medium' present since the 2024-03-31 release; mistral-medium-2312 was the only Mistral Medium model then (same mapping as the Arena lane).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "mistral-medium",
      "column_label": "score",
      "raw_value": "31.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.312,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'internlm2.5-20b-chat' score 31.2 (95% CI -2.4, 2.8; average #tokens 576); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "internlm2.5-20b-chat",
      "column_label": "score",
      "raw_value": "31.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-3-small-8k-instruct",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.298,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'phi-3-small-8k-instruct' score 29.8 (95% CI -1.8, 1.9; average #tokens 568); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "phi-3-small-8k-instruct",
      "column_label": "score",
      "raw_value": "29.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.248,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-3.5-turbo-0613' score 24.8 (95% CI -1.9, 2.3; average #tokens 401); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-3.5-turbo-0613",
      "column_label": "score",
      "raw_value": "24.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-2-0",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.24,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-2.0' score 24.0 (95% CI -1.8, 1.8; average #tokens 295); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-2.0",
      "column_label": "score",
      "raw_value": "24.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.234,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Mixtral-8x7B-Instruct-v0.1' score 23.4 (95% CI -2.0, 1.9; average #tokens 457); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Mixtral-8x7B-Instruct-v0.1",
      "column_label": "score",
      "raw_value": "23.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.233,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-3.5-turbo-0125' score 23.3 (95% CI -2.2, 1.9; average #tokens 329); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-3.5-turbo-0125",
      "column_label": "score",
      "raw_value": "23.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "yi-34b-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.231,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Yi-34B-Chat' score 23.1 (95% CI -1.6, 1.8; average #tokens 611); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Yi-34B-Chat",
      "column_label": "score",
      "raw_value": "23.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.23,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Starling-LM-7B-beta' score 23.0 (95% CI -1.8, 1.8; average #tokens 530); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Starling-LM-7B-beta",
      "column_label": "score",
      "raw_value": "23.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-2-1",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.228,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'claude-2.1' score 22.8 (95% CI -2.3, 1.8; average #tokens 290); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "claude-2.1",
      "column_label": "score",
      "raw_value": "22.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-1-8b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.213,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3.1-8b-instruct' score 21.3 (95% CI -1.9, 2.2; average #tokens 861); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3.1-8b-instruct",
      "column_label": "score",
      "raw_value": "21.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-8b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.206,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'llama-3-8b-instruct' score 20.6 (95% CI -2.0, 1.9; average #tokens 585); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "llama-3-8b-instruct",
      "column_label": "score",
      "raw_value": "20.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.189,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-3.5-turbo-1106' score 18.9 (95% CI -1.8, 1.6; average #tokens 285); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-3.5-turbo-1106",
      "column_label": "score",
      "raw_value": "18.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.181,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gpt-3.5-turbo-0301' score 18.1 (95% CI -1.9, 2.1; average #tokens 334); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gpt-3.5-turbo-0301",
      "column_label": "score",
      "raw_value": "18.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "snowflake-arctic-instruct",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.176,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'snowflake-arctic-instruct' score 17.6 (95% CI -1.8, 1.5; average #tokens 365); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "snowflake-arctic-instruct",
      "column_label": "score",
      "raw_value": "17.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "command-r-03-2024",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.17,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'command-r' score 17.0 (95% CI -1.7, 1.8; average #tokens 432); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label 'command-r' added 2024-05-21, before the 08-2024 release, so the 03-2024 checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "command-r",
      "column_label": "score",
      "raw_value": "17.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.154,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'phi-3-mini-128k-instruct' score 15.4 (95% CI -1.4, 1.4; average #tokens 609); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers). Label added 2024-05-21, before Microsoft's June 2024 update, so the April 2024 checkpoint.",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "phi-3-mini-128k-instruct",
      "column_label": "score",
      "raw_value": "15.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.15,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'tulu-2-dpo-70b' score 15.0 (95% CI -1.6, 1.3; average #tokens 550); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "tulu-2-dpo-70b",
      "column_label": "score",
      "raw_value": "15.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.128,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Starling-LM-7B-alpha' score 12.8 (95% CI -1.6, 1.4; average #tokens 483); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Starling-LM-7B-alpha",
      "column_label": "score",
      "raw_value": "12.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.121,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gemma-1.1-7b-it' score 12.1 (95% CI -1.3, 1.3; average #tokens 341); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gemma-1.1-7b-it",
      "column_label": "score",
      "raw_value": "12.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.116,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Llama-2-70b-chat-hf' score 11.6 (95% CI -1.5, 1.2; average #tokens 595); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Llama-2-70b-chat-hf",
      "column_label": "score",
      "raw_value": "11.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.086,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'vicuna-33b-v1.3' score 8.6 (95% CI -1.1, 1.1; average #tokens 451); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "vicuna-33b-v1.3",
      "column_label": "score",
      "raw_value": "8.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.075,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gemma-7b-it' score 7.5 (95% CI -1.2, 1.3; average #tokens 378); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gemma-7b-it",
      "column_label": "score",
      "raw_value": "7.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.046,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Llama-2-7b-chat-hf' score 4.6 (95% CI -0.8, 0.8; average #tokens 561); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "Llama-2-7b-chat-hf",
      "column_label": "score",
      "raw_value": "4.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.034,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gemma-1.1-2b-it' score 3.4 (95% CI -0.6, 0.8; average #tokens 316); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gemma-1.1-2b-it",
      "column_label": "score",
      "raw_value": "3.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.03,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "source_id": "arena-hard-auto-readme-v0-1",
    "by": "independent",
    "date": "2024-11-14",
    "mode": "instruct",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'gemma-2b-it' score 3.0 (95% CI -0.6, 0.6; average #tokens 369); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers (all README leaderboard edits committed by the maintainers).",
    "evidence": {
      "kind": "table",
      "table": "README 'Leaderboard' (no style control)",
      "row_label": "gemma-2b-it",
      "column_label": "score",
      "raw_value": "3.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "cruxeval-i",
    "score": 0.424875,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-13b_temp0.2_input.json', field 'pass_at_1' = 42.4875 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-13b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-13b_temp0.2_input.json",
      "row_label": "codellama-13b",
      "column_label": "pass_at_1",
      "raw_value": "42.4875",
      "scale_divisor": 100
    },
    "raw_value": 42.4875,
    "scale_divisor": 100
  },
  {
    "model": "codellama-13b",
    "bench": "cruxeval-o",
    "score": 0.397375,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-13b_temp0.2_output.json', field 'pass_at_1' = 39.7375 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-13b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-13b_temp0.2_output.json",
      "row_label": "codellama-13b",
      "column_label": "pass_at_1",
      "raw_value": "39.7375",
      "scale_divisor": 100
    },
    "raw_value": 39.7375,
    "scale_divisor": 100
  },
  {
    "model": "codellama-34b",
    "bench": "cruxeval-i",
    "score": 0.472125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-34b_temp0.2_input.json', field 'pass_at_1' = 47.2125 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-34b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-34b_temp0.2_input.json",
      "row_label": "codellama-34b",
      "column_label": "pass_at_1",
      "raw_value": "47.2125",
      "scale_divisor": 100
    },
    "raw_value": 47.2125,
    "scale_divisor": 100
  },
  {
    "model": "codellama-34b",
    "bench": "cruxeval-o",
    "score": 0.424,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-34b_temp0.2_output.json', field 'pass_at_1' = 42.39999999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-34b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-34b_temp0.2_output.json",
      "row_label": "codellama-34b",
      "column_label": "pass_at_1",
      "raw_value": "42.4",
      "scale_divisor": 100
    },
    "raw_value": 42.4,
    "scale_divisor": 100
  },
  {
    "model": "codellama-7b",
    "bench": "cruxeval-i",
    "score": 0.3595,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-7b_temp0.2_input.json', field 'pass_at_1' = 35.94999999999998 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-7b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-7b_temp0.2_input.json",
      "row_label": "codellama-7b",
      "column_label": "pass_at_1",
      "raw_value": "35.95",
      "scale_divisor": 100
    },
    "raw_value": 35.95,
    "scale_divisor": 100
  },
  {
    "model": "codellama-7b",
    "bench": "cruxeval-o",
    "score": 0.342125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-7b_temp0.2_output.json', field 'pass_at_1' = 34.212500000000006 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-7b-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-7b_temp0.2_output.json",
      "row_label": "codellama-7b",
      "column_label": "pass_at_1",
      "raw_value": "34.2125",
      "scale_divisor": 100
    },
    "raw_value": 34.2125,
    "scale_divisor": 100
  },
  {
    "model": "codellama-13b-python",
    "bench": "cruxeval-i",
    "score": 0.397375,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-13b_temp0.2_input.json', field 'pass_at_1' = 39.73749999999997 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-13b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-13b_temp0.2_input.json",
      "row_label": "codellama-python-13b",
      "column_label": "pass_at_1",
      "raw_value": "39.7375",
      "scale_divisor": 100
    },
    "raw_value": 39.7375,
    "scale_divisor": 100
  },
  {
    "model": "codellama-13b-python",
    "bench": "cruxeval-o",
    "score": 0.3975,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-13b_temp0.2_output.json', field 'pass_at_1' = 39.75000000000001 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-13b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-13b_temp0.2_output.json",
      "row_label": "codellama-python-13b",
      "column_label": "pass_at_1",
      "raw_value": "39.75",
      "scale_divisor": 100
    },
    "raw_value": 39.75,
    "scale_divisor": 100
  },
  {
    "model": "codellama-34b-python",
    "bench": "cruxeval-i",
    "score": 0.438625,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-34b_temp0.2_input.json', field 'pass_at_1' = 43.862499999999976 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-34b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-34b_temp0.2_input.json",
      "row_label": "codellama-python-34b",
      "column_label": "pass_at_1",
      "raw_value": "43.8625",
      "scale_divisor": 100
    },
    "raw_value": 43.8625,
    "scale_divisor": 100
  },
  {
    "model": "codellama-34b-python",
    "bench": "cruxeval-o",
    "score": 0.413875,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-34b_temp0.2_output.json', field 'pass_at_1' = 41.38750000000002 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-34b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-34b_temp0.2_output.json",
      "row_label": "codellama-python-34b",
      "column_label": "pass_at_1",
      "raw_value": "41.3875",
      "scale_divisor": 100
    },
    "raw_value": 41.3875,
    "scale_divisor": 100
  },
  {
    "model": "codellama-7b-python",
    "bench": "cruxeval-i",
    "score": 0.373,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-7b_temp0.2_input.json', field 'pass_at_1' = 37.3 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-7b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-7b_temp0.2_input.json",
      "row_label": "codellama-python-7b",
      "column_label": "pass_at_1",
      "raw_value": "37.3",
      "scale_divisor": 100
    },
    "raw_value": 37.3,
    "scale_divisor": 100
  },
  {
    "model": "codellama-7b-python",
    "bench": "cruxeval-o",
    "score": 0.359,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "lab",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/codellama-python-7b_temp0.2_output.json', field 'pass_at_1' = 35.9 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = codellama/CodeLlama-7b-Python-hf (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit). by: lab because CRUXEval is by Meta AI authors including Code Llama authors (Roziere, Synnaeve).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/codellama-python-7b_temp0.2_output.json",
      "row_label": "codellama-python-7b",
      "column_label": "pass_at_1",
      "raw_value": "35.9",
      "scale_divisor": 100
    },
    "raw_value": 35.9,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "cruxeval-i",
    "score": 0.27775,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-1.3b_temp0.2_input.json', field 'pass_at_1' = 27.775 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-1.3b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-1.3b_temp0.2_input.json",
      "row_label": "deepseek-base-1.3b",
      "column_label": "pass_at_1",
      "raw_value": "27.775",
      "scale_divisor": 100
    },
    "raw_value": 27.775,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "cruxeval-o",
    "score": 0.309875,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-1.3b_temp0.2_output.json', field 'pass_at_1' = 30.98749999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-1.3b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-1.3b_temp0.2_output.json",
      "row_label": "deepseek-base-1.3b",
      "column_label": "pass_at_1",
      "raw_value": "30.9875",
      "scale_divisor": 100
    },
    "raw_value": 30.9875,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "cruxeval-i",
    "score": 0.465125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-33b_temp0.2_input.json', field 'pass_at_1' = 46.51250000000002 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-33b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-33b_temp0.2_input.json",
      "row_label": "deepseek-base-33b",
      "column_label": "pass_at_1",
      "raw_value": "46.5125",
      "scale_divisor": 100
    },
    "raw_value": 46.5125,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "cruxeval-o",
    "score": 0.486,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-33b_temp0.2_output.json', field 'pass_at_1' = 48.599999999999994 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-33b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-33b_temp0.2_output.json",
      "row_label": "deepseek-base-33b",
      "column_label": "pass_at_1",
      "raw_value": "48.6",
      "scale_divisor": 100
    },
    "raw_value": 48.6,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "cruxeval-i",
    "score": 0.418625,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-6.7b_temp0.2_input.json', field 'pass_at_1' = 41.86250000000001 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-6.7b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-6.7b_temp0.2_input.json",
      "row_label": "deepseek-base-6.7b",
      "column_label": "pass_at_1",
      "raw_value": "41.8625",
      "scale_divisor": 100
    },
    "raw_value": 41.8625,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "cruxeval-o",
    "score": 0.434625,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-base-6.7b_temp0.2_output.json', field 'pass_at_1' = 43.46249999999998 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-6.7b-base (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-base-6.7b_temp0.2_output.json",
      "row_label": "deepseek-base-6.7b",
      "column_label": "pass_at_1",
      "raw_value": "43.4625",
      "scale_divisor": 100
    },
    "raw_value": 43.4625,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "cruxeval-i",
    "score": 0.271625,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-1.3b_temp0.2_input.json', field 'pass_at_1' = 27.162499999999994 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-1.3b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-1.3b_temp0.2_input.json",
      "row_label": "deepseek-instruct-1.3b",
      "column_label": "pass_at_1",
      "raw_value": "27.1625",
      "scale_divisor": 100
    },
    "raw_value": 27.1625,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "cruxeval-o",
    "score": 0.286625,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-1.3b_temp0.2_output.json', field 'pass_at_1' = 28.66250000000001 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-1.3b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-1.3b_temp0.2_output.json",
      "row_label": "deepseek-instruct-1.3b",
      "column_label": "pass_at_1",
      "raw_value": "28.6625",
      "scale_divisor": 100
    },
    "raw_value": 28.6625,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "cruxeval-i",
    "score": 0.465,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-33b_temp0.2_input.json', field 'pass_at_1' = 46.50000000000001 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-33b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-33b_temp0.2_input.json",
      "row_label": "deepseek-instruct-33b",
      "column_label": "pass_at_1",
      "raw_value": "46.5",
      "scale_divisor": 100
    },
    "raw_value": 46.5,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "cruxeval-o",
    "score": 0.499,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-33b_temp0.2_output.json', field 'pass_at_1' = 49.90000000000001 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-33b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-33b_temp0.2_output.json",
      "row_label": "deepseek-instruct-33b",
      "column_label": "pass_at_1",
      "raw_value": "49.9",
      "scale_divisor": 100
    },
    "raw_value": 49.9,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "cruxeval-i",
    "score": 0.37375,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-6.7b_temp0.2_input.json', field 'pass_at_1' = 37.375000000000014 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-6.7b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-6.7b_temp0.2_input.json",
      "row_label": "deepseek-instruct-6.7b",
      "column_label": "pass_at_1",
      "raw_value": "37.375",
      "scale_divisor": 100
    },
    "raw_value": 37.375,
    "scale_divisor": 100
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "cruxeval-o",
    "score": 0.41225,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/deepseek-instruct-6.7b_temp0.2_output.json', field 'pass_at_1' = 41.224999999999994 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = deepseek-ai/deepseek-coder-6.7b-instruct (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/deepseek-instruct-6.7b_temp0.2_output.json",
      "row_label": "deepseek-instruct-6.7b",
      "column_label": "pass_at_1",
      "raw_value": "41.225",
      "scale_divisor": 100
    },
    "raw_value": 41.225,
    "scale_divisor": 100
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "cruxeval-i",
    "score": 0.490375,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/gpt-3.5-turbo-0613_temp0.2_input.json', field 'pass_at_1' = 49.03749999999999 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = OpenAI API gpt-3.5-turbo-0613 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/gpt-3.5-turbo-0613_temp0.2_input.json",
      "row_label": "gpt-3.5-turbo-0613",
      "column_label": "pass_at_1",
      "raw_value": "49.0375",
      "scale_divisor": 100
    },
    "raw_value": 49.0375,
    "scale_divisor": 100
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "cruxeval-o",
    "score": 0.494125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/gpt-3.5-turbo-0613_temp0.2_output.json', field 'pass_at_1' = 49.412499999999994 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = OpenAI API gpt-3.5-turbo-0613 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/gpt-3.5-turbo-0613_temp0.2_output.json",
      "row_label": "gpt-3.5-turbo-0613",
      "column_label": "pass_at_1",
      "raw_value": "49.4125",
      "scale_divisor": 100
    },
    "raw_value": 49.4125,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-0613",
    "bench": "cruxeval-i",
    "score": 0.698125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/gpt-4-0613_temp0.2_input.json', field 'pass_at_1' = 69.81249999999999 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = OpenAI API gpt-4-0613 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/gpt-4-0613_temp0.2_input.json",
      "row_label": "gpt-4-0613",
      "column_label": "pass_at_1",
      "raw_value": "69.8125",
      "scale_divisor": 100
    },
    "raw_value": 69.8125,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-0613",
    "bench": "cruxeval-o",
    "score": 0.687,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/gpt-4-0613_temp0.2_output.json', field 'pass_at_1' = 68.69999999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = OpenAI API gpt-4-0613 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/gpt-4-0613_temp0.2_output.json",
      "row_label": "gpt-4-0613",
      "column_label": "pass_at_1",
      "raw_value": "68.7",
      "scale_divisor": 100
    },
    "raw_value": 68.7,
    "scale_divisor": 100
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "cruxeval-i",
    "score": 0.417125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/magicoder-ds-7b_temp0.2_input.json', field 'pass_at_1' = 41.71249999999999 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = ise-uiuc/Magicoder-S-DS-6.7B (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/magicoder-ds-7b_temp0.2_input.json",
      "row_label": "magicoder-ds-7b",
      "column_label": "pass_at_1",
      "raw_value": "41.7125",
      "scale_divisor": 100
    },
    "raw_value": 41.7125,
    "scale_divisor": 100
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "cruxeval-o",
    "score": 0.444125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/magicoder-ds-7b_temp0.2_output.json', field 'pass_at_1' = 44.41250000000001 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = ise-uiuc/Magicoder-S-DS-6.7B (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/magicoder-ds-7b_temp0.2_output.json",
      "row_label": "magicoder-ds-7b",
      "column_label": "pass_at_1",
      "raw_value": "44.4125",
      "scale_divisor": 100
    },
    "raw_value": 44.4125,
    "scale_divisor": 100
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "cruxeval-i",
    "score": 0.393,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/mixtral-8x7b_temp0.2_input.json', field 'pass_at_1' = 39.29999999999998 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = mistralai/Mixtral-8x7B-v0.1 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/mixtral-8x7b_temp0.2_input.json",
      "row_label": "mixtral-8x7b",
      "column_label": "pass_at_1",
      "raw_value": "39.3",
      "scale_divisor": 100
    },
    "raw_value": 39.3,
    "scale_divisor": 100
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "cruxeval-o",
    "score": 0.404875,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/mixtral-8x7b_temp0.2_output.json', field 'pass_at_1' = 40.4875 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = mistralai/Mixtral-8x7B-v0.1 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/mixtral-8x7b_temp0.2_output.json",
      "row_label": "mixtral-8x7b",
      "column_label": "pass_at_1",
      "raw_value": "40.4875",
      "scale_divisor": 100
    },
    "raw_value": 40.4875,
    "scale_divisor": 100
  },
  {
    "model": "phi-1",
    "bench": "cruxeval-i",
    "score": 0.131,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phi-1_temp0.2_input.json', field 'pass_at_1' = 13.100000000000001 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = microsoft/phi-1 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phi-1_temp0.2_input.json",
      "row_label": "phi-1",
      "column_label": "pass_at_1",
      "raw_value": "13.1",
      "scale_divisor": 100
    },
    "raw_value": 13.1,
    "scale_divisor": 100
  },
  {
    "model": "phi-1",
    "bench": "cruxeval-o",
    "score": 0.21675,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phi-1_temp0.2_output.json', field 'pass_at_1' = 21.675 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = microsoft/phi-1 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phi-1_temp0.2_output.json",
      "row_label": "phi-1",
      "column_label": "pass_at_1",
      "raw_value": "21.675",
      "scale_divisor": 100
    },
    "raw_value": 21.675,
    "scale_divisor": 100
  },
  {
    "model": "phi-2",
    "bench": "cruxeval-i",
    "score": 0.316125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phi-2_temp0.2_input.json', field 'pass_at_1' = 31.612499999999976 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = microsoft/phi-2 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phi-2_temp0.2_input.json",
      "row_label": "phi-2",
      "column_label": "pass_at_1",
      "raw_value": "31.6125",
      "scale_divisor": 100
    },
    "raw_value": 31.6125,
    "scale_divisor": 100
  },
  {
    "model": "phi-2",
    "bench": "cruxeval-o",
    "score": 0.33525,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phi-2_temp0.2_output.json', field 'pass_at_1' = 33.52499999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = microsoft/phi-2 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phi-2_temp0.2_output.json",
      "row_label": "phi-2",
      "column_label": "pass_at_1",
      "raw_value": "33.525",
      "scale_divisor": 100
    },
    "raw_value": 33.525,
    "scale_divisor": 100
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "cruxeval-i",
    "score": 0.472,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phind_temp0.2_input.json', field 'pass_at_1' = 47.20000000000001 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = Phind/Phind-CodeLlama-34B-v2 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phind_temp0.2_input.json",
      "row_label": "phind",
      "column_label": "pass_at_1",
      "raw_value": "47.2",
      "scale_divisor": 100
    },
    "raw_value": 47.2,
    "scale_divisor": 100
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "cruxeval-o",
    "score": 0.397125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/phind_temp0.2_output.json', field 'pass_at_1' = 39.712500000000006 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = Phind/Phind-CodeLlama-34B-v2 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/phind_temp0.2_output.json",
      "row_label": "phind",
      "column_label": "pass_at_1",
      "raw_value": "39.7125",
      "scale_divisor": 100
    },
    "raw_value": 39.7125,
    "scale_divisor": 100
  },
  {
    "model": "starcoderbase",
    "bench": "cruxeval-i",
    "score": 0.31325,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/starcoderbase-16b_temp0.2_input.json', field 'pass_at_1' = 31.325000000000003 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = bigcode/starcoderbase (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/starcoderbase-16b_temp0.2_input.json",
      "row_label": "starcoderbase-16b",
      "column_label": "pass_at_1",
      "raw_value": "31.325",
      "scale_divisor": 100
    },
    "raw_value": 31.325,
    "scale_divisor": 100
  },
  {
    "model": "starcoderbase",
    "bench": "cruxeval-o",
    "score": 0.34175,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/starcoderbase-16b_temp0.2_output.json', field 'pass_at_1' = 34.17499999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = bigcode/starcoderbase (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/starcoderbase-16b_temp0.2_output.json",
      "row_label": "starcoderbase-16b",
      "column_label": "pass_at_1",
      "raw_value": "34.175",
      "scale_divisor": 100
    },
    "raw_value": 34.175,
    "scale_divisor": 100
  },
  {
    "model": "starcoderbase-7b",
    "bench": "cruxeval-i",
    "score": 0.297375,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/starcoderbase-7b_temp0.2_input.json', field 'pass_at_1' = 29.73749999999999 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = bigcode/starcoderbase-7b (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/starcoderbase-7b_temp0.2_input.json",
      "row_label": "starcoderbase-7b",
      "column_label": "pass_at_1",
      "raw_value": "29.7375",
      "scale_divisor": 100
    },
    "raw_value": 29.7375,
    "scale_divisor": 100
  },
  {
    "model": "starcoderbase-7b",
    "bench": "cruxeval-o",
    "score": 0.322,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "base",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/starcoderbase-7b_temp0.2_output.json', field 'pass_at_1' = 32.19999999999999 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = bigcode/starcoderbase-7b (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/starcoderbase-7b_temp0.2_output.json",
      "row_label": "starcoderbase-7b",
      "column_label": "pass_at_1",
      "raw_value": "32.2",
      "scale_divisor": 100
    },
    "raw_value": 32.2,
    "scale_divisor": 100
  },
  {
    "model": "wizardcoder-python-13b-v1-0",
    "bench": "cruxeval-i",
    "score": 0.36475,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/wizard-13b_temp0.2_input.json', field 'pass_at_1' = 36.475000000000016 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = WizardLM/WizardCoder-Python-13B-V1.0 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/wizard-13b_temp0.2_input.json",
      "row_label": "wizard-13b",
      "column_label": "pass_at_1",
      "raw_value": "36.475",
      "scale_divisor": 100
    },
    "raw_value": 36.475,
    "scale_divisor": 100
  },
  {
    "model": "wizardcoder-python-13b-v1-0",
    "bench": "cruxeval-o",
    "score": 0.41275,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/wizard-13b_temp0.2_output.json', field 'pass_at_1' = 41.27500000000002 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = WizardLM/WizardCoder-Python-13B-V1.0 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/wizard-13b_temp0.2_output.json",
      "row_label": "wizard-13b",
      "column_label": "pass_at_1",
      "raw_value": "41.275",
      "scale_divisor": 100
    },
    "raw_value": 41.275,
    "scale_divisor": 100
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "cruxeval-i",
    "score": 0.42675,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/wizard-34b_temp0.2_input.json', field 'pass_at_1' = 42.675 (800 problems, divided by 100). CRUXEval-I direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = WizardLM/WizardCoder-Python-34B-V1.0 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/wizard-34b_temp0.2_input.json",
      "row_label": "wizard-34b",
      "column_label": "pass_at_1",
      "raw_value": "42.675",
      "scale_divisor": 100
    },
    "raw_value": 42.675,
    "scale_divisor": 100
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "cruxeval-o",
    "score": 0.434125,
    "source": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/samples/evaluation_results.zip",
    "source_id": "cruxeval-published-results",
    "by": "independent",
    "date": "2024-01-03",
    "mode": "instruct",
    "note": "CRUXEval repository (MIT) at commit 190faf16d1, samples/evaluation_results.zip member 'evaluation_results/wizard-34b_temp0.2_output.json', field 'pass_at_1' = 43.41250000000001 (800 problems, divided by 100). CRUXEval-O direct prediction (no CoT), temperature 0.2, 10 samples per problem, pass@1 as in the CRUXEval paper; generations and scores produced by the CRUXEval authors (README: 'We also open-source generations and outputs for the models we display on the leaderboard'). Model = WizardLM/WizardCoder-Python-34B-V1.0 (inference/scripts/run_input_prediction.sh / openai/openai_run.py in the same commit).",
    "evidence": {
      "kind": "table",
      "table": "zip member evaluation_results/wizard-34b_temp0.2_output.json",
      "row_label": "wizard-34b",
      "column_label": "pass_at_1",
      "raw_value": "43.4125",
      "scale_divisor": 100
    },
    "raw_value": 43.4125,
    "scale_divisor": 100
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "humaneval",
    "score": 0.835,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'GPT-4-Turbo NA' (Table 5 model ID 'gpt-4-0125-preview'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 83.5 (the bracketed HumanEval+ value 80.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "GPT-4-Turbo NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "83.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "evoeval",
    "score": 0.651,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'GPT-4-Turbo NA' (Table 5 model ID 'gpt-4-0125-preview'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 65.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 50.0, 'Creative': 61.0, 'Subtle': 82.0, 'Combine': 45.0, 'Tool_Use': 69.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "GPT-4-Turbo NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "65.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 50,
      "Creative": 61,
      "Subtle": 82,
      "Combine": 45,
      "Tool_Use": 69
    }
  },
  {
    "model": "gpt-4-0613",
    "bench": "humaneval",
    "score": 0.823,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'GPT-4 NA' (Table 5 model ID 'gpt-4-0613'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 82.3 (the bracketed HumanEval+ value 76.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "GPT-4 NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "82.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-4-0613",
    "bench": "evoeval",
    "score": 0.662,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'GPT-4 NA' (Table 5 model ID 'gpt-4-0613'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 66.2: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 52.0, 'Creative': 66.0, 'Subtle': 76.0, 'Combine': 53.0, 'Tool_Use': 68.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "GPT-4 NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "66.2",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 52,
      "Creative": 66,
      "Subtle": 76,
      "Combine": 53,
      "Tool_Use": 68
    }
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'ChatGPT NA' (Table 5 model ID 'gpt-3.5-turbo-0125'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 76.8 (the bracketed HumanEval+ value 69.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "ChatGPT NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "evoeval",
    "score": 0.531,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'ChatGPT NA' (Table 5 model ID 'gpt-3.5-turbo-0125'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 53.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 33.0, 'Creative': 42.0, 'Subtle': 70.0, 'Combine': 33.0, 'Tool_Use': 64.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "ChatGPT NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "53.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 33,
      "Creative": 42,
      "Subtle": 70,
      "Combine": 33,
      "Tool_Use": 64
    }
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "humaneval",
    "score": 0.823,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-3 NA' (Table 5 model ID 'claude-3-opus-20240229'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 82.3 (the bracketed HumanEval+ value 75.0 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-3 NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "82.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "evoeval",
    "score": 0.629,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-3 NA' (Table 5 model ID 'claude-3-opus-20240229'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 62.9: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 50.0, 'Creative': 53.0, 'Subtle': 81.0, 'Combine': 42.0, 'Tool_Use': 69.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-3 NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "62.9",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 50,
      "Creative": 53,
      "Subtle": 81,
      "Combine": 42,
      "Tool_Use": 69
    }
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "humaneval",
    "score": 0.744,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-3-haiku NA' (Table 5 model ID 'claude-3-haiku-20240307'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 74.4 (the bracketed HumanEval+ value 66.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-3-haiku NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "evoeval",
    "score": 0.521,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-3-haiku NA' (Table 5 model ID 'claude-3-haiku-20240307'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 52.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 40.0, 'Creative': 47.0, 'Subtle': 65.0, 'Combine': 25.0, 'Tool_Use': 61.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-3-haiku NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "52.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 40,
      "Creative": 47,
      "Subtle": 65,
      "Combine": 25,
      "Tool_Use": 61
    }
  },
  {
    "model": "claude-2-1",
    "bench": "humaneval",
    "score": 0.665,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-2 NA' (Table 5 model ID 'claude-2.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 66.5 (the bracketed HumanEval+ value 62.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-2 NA",
      "column_label": "HumanEval pass@1",
      "raw_value": "66.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-2-1",
    "bench": "evoeval",
    "score": 0.462,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Claude-2 NA' (Table 5 model ID 'claude-2.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 46.2: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 29.0, 'Creative': 42.0, 'Subtle': 64.0, 'Combine': 19.0, 'Tool_Use': 57.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Claude-2 NA",
      "column_label": "EvoEval pass@1",
      "raw_value": "46.2",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 29,
      "Creative": 42,
      "Subtle": 64,
      "Combine": 19,
      "Tool_Use": 57
    }
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "humaneval",
    "score": 0.78,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 33b' (Table 5 model ID 'deepseek-ai/deepseek-coder-33b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 78.0 (the bracketed HumanEval+ value 73.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 33b",
      "column_label": "HumanEval pass@1",
      "raw_value": "78.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "evoeval",
    "score": 0.56,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 33b' (Table 5 model ID 'deepseek-ai/deepseek-coder-33b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 56.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 47.0, 'Creative': 47.0, 'Subtle': 67.0, 'Combine': 31.0, 'Tool_Use': 66.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 33b",
      "column_label": "EvoEval pass@1",
      "raw_value": "56.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 47,
      "Creative": 47,
      "Subtle": 67,
      "Combine": 31,
      "Tool_Use": 66
    }
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "humaneval",
    "score": 0.744,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 6.7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-6.7b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 74.4 (the bracketed HumanEval+ value 69.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 6.7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "74.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "evoeval",
    "score": 0.469,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 6.7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-6.7b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 46.9: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 40.0, 'Creative': 37.0, 'Subtle': 61.0, 'Combine': 18.0, 'Tool_Use': 51.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 6.7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "46.9",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 40,
      "Creative": 37,
      "Subtle": 61,
      "Combine": 18,
      "Tool_Use": 51
    }
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "humaneval",
    "score": 0.634,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 1.3b' (Table 5 model ID 'deepseek-ai/deepseek-coder-1.3b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 63.4 (the bracketed HumanEval+ value 60.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 1.3b",
      "column_label": "HumanEval pass@1",
      "raw_value": "63.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "evoeval",
    "score": 0.349,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-Inst 1.3b' (Table 5 model ID 'deepseek-ai/deepseek-coder-1.3b-instruct'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 34.9: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 20.0, 'Creative': 25.0, 'Subtle': 53.0, 'Combine': 9.0, 'Tool_Use': 39.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-Inst 1.3b",
      "column_label": "EvoEval pass@1",
      "raw_value": "34.9",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 20,
      "Creative": 25,
      "Subtle": 53,
      "Combine": 9,
      "Tool_Use": 39
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "humaneval",
    "score": 0.506,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 33b' (Table 5 model ID 'deepseek-ai/deepseek-coder-33b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 50.6 (the bracketed HumanEval+ value 42.7 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 33b",
      "column_label": "HumanEval pass@1",
      "raw_value": "50.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "evoeval",
    "score": 0.368,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 33b' (Table 5 model ID 'deepseek-ai/deepseek-coder-33b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 36.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 26.0, 'Creative': 23.0, 'Subtle': 47.0, 'Combine': 11.0, 'Tool_Use': 63.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 33b",
      "column_label": "EvoEval pass@1",
      "raw_value": "36.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 26,
      "Creative": 23,
      "Subtle": 47,
      "Combine": 11,
      "Tool_Use": 63
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "humaneval",
    "score": 0.451,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 6.7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-6.7b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 45.1 (the bracketed HumanEval+ value 38.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 6.7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "45.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "evoeval",
    "score": 0.329,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 6.7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-6.7b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 32.9: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 21.0, 'Creative': 24.0, 'Subtle': 47.0, 'Combine': 5.0, 'Tool_Use': 55.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 6.7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "32.9",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 21,
      "Creative": 24,
      "Subtle": 47,
      "Combine": 5,
      "Tool_Use": 55
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "humaneval",
    "score": 0.299,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 1.3b' (Table 5 model ID 'deepseek-ai/deepseek-coder-1.3b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 29.9 (the bracketed HumanEval+ value 26.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 1.3b",
      "column_label": "HumanEval pass@1",
      "raw_value": "29.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "evoeval",
    "score": 0.203,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker 1.3b' (Table 5 model ID 'deepseek-ai/deepseek-coder-1.3b-base'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 20.3: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 6.0, 'Creative': 19.0, 'Subtle': 27.0, 'Combine': 0.0, 'Tool_Use': 40.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker 1.3b",
      "column_label": "EvoEval pass@1",
      "raw_value": "20.3",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 6,
      "Creative": 19,
      "Subtle": 27,
      "Combine": 0,
      "Tool_Use": 40
    }
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "humaneval",
    "score": 0.689,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-1.5-Inst 7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 68.9 (the bracketed HumanEval+ value 63.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-1.5-Inst 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "68.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "evoeval",
    "score": 0.488,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-1.5-Inst 7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 48.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 37.0, 'Creative': 37.0, 'Subtle': 66.0, 'Combine': 24.0, 'Tool_Use': 60.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-1.5-Inst 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "48.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 37,
      "Creative": 37,
      "Subtle": 66,
      "Combine": 24,
      "Tool_Use": 60
    }
  },
  {
    "model": "deepseek-coder-7b-base-v1-5",
    "bench": "humaneval",
    "score": 0.421,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-1.5 7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-7b-base-v1.5'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 42.1 (the bracketed HumanEval+ value 34.8 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-1.5 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "42.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-7b-base-v1-5",
    "bench": "evoeval",
    "score": 0.33,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'DeepSeeker-1.5 7b' (Table 5 model ID 'deepseek-ai/deepseek-coder-7b-base-v1.5'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 33.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 21.0, 'Creative': 34.0, 'Subtle': 43.0, 'Combine': 4.0, 'Tool_Use': 54.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "DeepSeeker-1.5 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "33.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 21,
      "Creative": 34,
      "Subtle": 43,
      "Combine": 4,
      "Tool_Use": 54
    }
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humaneval",
    "score": 0.665,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 70b' (Table 5 model ID 'codellama/CodeLlama-70b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 66.5 (the bracketed HumanEval+ value 59.8 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 70b",
      "column_label": "HumanEval pass@1",
      "raw_value": "66.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "evoeval",
    "score": 0.477,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 70b' (Table 5 model ID 'codellama/CodeLlama-70b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 47.7: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 31.0, 'Creative': 41.0, 'Subtle': 65.0, 'Combine': 18.0, 'Tool_Use': 65.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 70b",
      "column_label": "EvoEval pass@1",
      "raw_value": "47.7",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 31,
      "Creative": 41,
      "Subtle": 65,
      "Combine": 18,
      "Tool_Use": 65
    }
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "humaneval",
    "score": 0.518,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 34b' (Table 5 model ID 'codellama/CodeLlama-34b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 51.8 (the bracketed HumanEval+ value 43.9 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 34b",
      "column_label": "HumanEval pass@1",
      "raw_value": "51.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "evoeval",
    "score": 0.333,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 34b' (Table 5 model ID 'codellama/CodeLlama-34b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 33.3: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 22.0, 'Creative': 27.0, 'Subtle': 43.0, 'Combine': 9.0, 'Tool_Use': 47.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 34b",
      "column_label": "EvoEval pass@1",
      "raw_value": "33.3",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 22,
      "Creative": 27,
      "Subtle": 43,
      "Combine": 9,
      "Tool_Use": 47
    }
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "humaneval",
    "score": 0.488,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 13b' (Table 5 model ID 'codellama/CodeLlama-13b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 48.8 (the bracketed HumanEval+ value 42.7 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 13b",
      "column_label": "HumanEval pass@1",
      "raw_value": "48.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "evoeval",
    "score": 0.338,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 13b' (Table 5 model ID 'codellama/CodeLlama-13b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 33.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 21.0, 'Creative': 25.0, 'Subtle': 46.0, 'Combine': 8.0, 'Tool_Use': 54.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 13b",
      "column_label": "EvoEval pass@1",
      "raw_value": "33.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 21,
      "Creative": 25,
      "Subtle": 46,
      "Combine": 8,
      "Tool_Use": 54
    }
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "humaneval",
    "score": 0.433,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 7b' (Table 5 model ID 'codellama/CodeLlama-7b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 43.3 (the bracketed HumanEval+ value 39.0 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "evoeval",
    "score": 0.279,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama-Inst 7b' (Table 5 model ID 'codellama/CodeLlama-7b-Instruct-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 27.9: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 14.0, 'Creative': 18.0, 'Subtle': 40.0, 'Combine': 8.0, 'Tool_Use': 44.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama-Inst 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "27.9",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 14,
      "Creative": 18,
      "Subtle": 40,
      "Combine": 8,
      "Tool_Use": 44
    }
  },
  {
    "model": "codellama-70b-python",
    "bench": "humaneval",
    "score": 0.604,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 70b' (Table 5 model ID 'codellama/CodeLlama-70b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 60.4 (the bracketed HumanEval+ value 52.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 70b",
      "column_label": "HumanEval pass@1",
      "raw_value": "60.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-python",
    "bench": "evoeval",
    "score": 0.401,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 70b' (Table 5 model ID 'codellama/CodeLlama-70b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 40.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 25.0, 'Creative': 29.0, 'Subtle': 49.0, 'Combine': 14.0, 'Tool_Use': 63.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 70b",
      "column_label": "EvoEval pass@1",
      "raw_value": "40.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 25,
      "Creative": 29,
      "Subtle": 49,
      "Combine": 14,
      "Tool_Use": 63
    }
  },
  {
    "model": "codellama-34b-python",
    "bench": "humaneval",
    "score": 0.524,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 34b' (Table 5 model ID 'codellama/CodeLlama-34b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 52.4 (the bracketed HumanEval+ value 43.3 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 34b",
      "column_label": "HumanEval pass@1",
      "raw_value": "52.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-python",
    "bench": "evoeval",
    "score": 0.322,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 34b' (Table 5 model ID 'codellama/CodeLlama-34b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 32.2: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 15.0, 'Creative': 24.0, 'Subtle': 47.0, 'Combine': 11.0, 'Tool_Use': 44.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 34b",
      "column_label": "EvoEval pass@1",
      "raw_value": "32.2",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 15,
      "Creative": 24,
      "Subtle": 47,
      "Combine": 11,
      "Tool_Use": 44
    }
  },
  {
    "model": "codellama-13b-python",
    "bench": "humaneval",
    "score": 0.427,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 13b' (Table 5 model ID 'codellama/CodeLlama-13b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 42.7 (the bracketed HumanEval+ value 36.6 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 13b",
      "column_label": "HumanEval pass@1",
      "raw_value": "42.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-python",
    "bench": "evoeval",
    "score": 0.294,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 13b' (Table 5 model ID 'codellama/CodeLlama-13b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 29.4: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 18.0, 'Creative': 24.0, 'Subtle': 38.0, 'Combine': 6.0, 'Tool_Use': 48.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 13b",
      "column_label": "EvoEval pass@1",
      "raw_value": "29.4",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 18,
      "Creative": 24,
      "Subtle": 38,
      "Combine": 6,
      "Tool_Use": 48
    }
  },
  {
    "model": "codellama-7b-python",
    "bench": "humaneval",
    "score": 0.396,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 7b' (Table 5 model ID 'codellama/CodeLlama-7b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 39.6 (the bracketed HumanEval+ value 36.6 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "39.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-python",
    "bench": "evoeval",
    "score": 0.256,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'CodeLlama 7b' (Table 5 model ID 'codellama/CodeLlama-7b-Python-hf'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 25.6: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 10.0, 'Creative': 15.0, 'Subtle': 42.0, 'Combine': 3.0, 'Tool_Use': 44.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "CodeLlama 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "25.6",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 10,
      "Creative": 15,
      "Subtle": 42,
      "Combine": 3,
      "Tool_Use": 44
    }
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "humaneval",
    "score": 0.616,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'WizardCoder 34b' (Table 5 model ID 'WizardLM/WizardCoder-Python-34B-V1.0'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 61.6 (the bracketed HumanEval+ value 54.3 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "WizardCoder 34b",
      "column_label": "HumanEval pass@1",
      "raw_value": "61.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "evoeval",
    "score": 0.408,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'WizardCoder 34b' (Table 5 model ID 'WizardLM/WizardCoder-Python-34B-V1.0'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 40.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 24.0, 'Creative': 32.0, 'Subtle': 55.0, 'Combine': 17.0, 'Tool_Use': 55.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "WizardCoder 34b",
      "column_label": "EvoEval pass@1",
      "raw_value": "40.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 24,
      "Creative": 32,
      "Subtle": 55,
      "Combine": 17,
      "Tool_Use": 55
    }
  },
  {
    "model": "wizardcoder-33b-v1-1",
    "bench": "humaneval",
    "score": 0.738,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'WizardCoder-1.1 33b' (Table 5 model ID 'WizardLM/WizardCoder-33B-V1.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 73.8 (the bracketed HumanEval+ value 69.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "WizardCoder-1.1 33b",
      "column_label": "HumanEval pass@1",
      "raw_value": "73.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "wizardcoder-33b-v1-1",
    "bench": "evoeval",
    "score": 0.533,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'WizardCoder-1.1 33b' (Table 5 model ID 'WizardLM/WizardCoder-33B-V1.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 53.3: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 48.0, 'Creative': 48.0, 'Subtle': 66.0, 'Combine': 20.0, 'Tool_Use': 64.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "WizardCoder-1.1 33b",
      "column_label": "EvoEval pass@1",
      "raw_value": "53.3",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 48,
      "Creative": 48,
      "Subtle": 66,
      "Combine": 20,
      "Tool_Use": 64
    }
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "humaneval",
    "score": 0.707,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Phind-CodeLlama-2 34b' (Table 5 model ID 'Phind/Phind-CodeLlama-34B-v2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 70.7 (the bracketed HumanEval+ value 66.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Phind-CodeLlama-2 34b",
      "column_label": "HumanEval pass@1",
      "raw_value": "70.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "evoeval",
    "score": 0.456,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Phind-CodeLlama-2 34b' (Table 5 model ID 'Phind/Phind-CodeLlama-34B-v2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 45.6: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 22.0, 'Creative': 35.0, 'Subtle': 63.0, 'Combine': 25.0, 'Tool_Use': 58.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Phind-CodeLlama-2 34b",
      "column_label": "EvoEval pass@1",
      "raw_value": "45.6",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 22,
      "Creative": 35,
      "Subtle": 63,
      "Combine": 25,
      "Tool_Use": 58
    }
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "lab",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Magicoder-s-DS 6.7b' (Table 5 model ID 'ise-uiuc/Magicoder-S-DS-6.7B'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). by: lab (Magicoder is by the same UIUC group). Column 'HumanEval' pass@1 = 76.8 (the bracketed HumanEval+ value 70.7 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Magicoder-s-DS 6.7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "76.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "evoeval",
    "score": 0.5,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "lab",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Magicoder-s-DS 6.7b' (Table 5 model ID 'ise-uiuc/Magicoder-S-DS-6.7B'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). by: lab (Magicoder is by the same UIUC group). Column 'EvoEval' pass@1 = 50.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 40.0, 'Creative': 34.0, 'Subtle': 67.0, 'Combine': 21.0, 'Tool_Use': 61.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Magicoder-s-DS 6.7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "50.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 40,
      "Creative": 34,
      "Subtle": 67,
      "Combine": 21,
      "Tool_Use": 61
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "humaneval",
    "score": 0.451,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 15b' (Table 5 model ID 'bigcode/starcoder2-15b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 45.1 (the bracketed HumanEval+ value 36.0 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 15b",
      "column_label": "HumanEval pass@1",
      "raw_value": "45.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "evoeval",
    "score": 0.29,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 15b' (Table 5 model ID 'bigcode/starcoder2-15b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 29.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 16.0, 'Creative': 19.0, 'Subtle': 41.0, 'Combine': 5.0, 'Tool_Use': 48.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 15b",
      "column_label": "EvoEval pass@1",
      "raw_value": "29.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 16,
      "Creative": 19,
      "Subtle": 41,
      "Combine": 5,
      "Tool_Use": 48
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "humaneval",
    "score": 0.348,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 7b' (Table 5 model ID 'bigcode/starcoder2-7b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 34.8 (the bracketed HumanEval+ value 31.1 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "34.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "evoeval",
    "score": 0.25,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 7b' (Table 5 model ID 'bigcode/starcoder2-7b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 25.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 12.0, 'Creative': 17.0, 'Subtle': 38.0, 'Combine': 2.0, 'Tool_Use': 46.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "25.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 12,
      "Creative": 17,
      "Subtle": 38,
      "Combine": 2,
      "Tool_Use": 46
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "humaneval",
    "score": 0.311,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 3b' (Table 5 model ID 'bigcode/starcoder2-3b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 31.1 (the bracketed HumanEval+ value 26.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 3b",
      "column_label": "HumanEval pass@1",
      "raw_value": "31.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "evoeval",
    "score": 0.202,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder2 3b' (Table 5 model ID 'bigcode/starcoder2-3b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 20.2: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 8.0, 'Creative': 14.0, 'Subtle': 31.0, 'Combine': 2.0, 'Tool_Use': 35.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder2 3b",
      "column_label": "EvoEval pass@1",
      "raw_value": "20.2",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 8,
      "Creative": 14,
      "Subtle": 31,
      "Combine": 2,
      "Tool_Use": 35
    }
  },
  {
    "model": "starcoder",
    "bench": "humaneval",
    "score": 0.348,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder 15b' (Table 5 model ID 'bigcode/starcoder'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 34.8 (the bracketed HumanEval+ value 30.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder 15b",
      "column_label": "HumanEval pass@1",
      "raw_value": "34.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder",
    "bench": "evoeval",
    "score": 0.235,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'StarCoder 15b' (Table 5 model ID 'bigcode/starcoder'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 23.5: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 12.0, 'Creative': 11.0, 'Subtle': 37.0, 'Combine': 2.0, 'Tool_Use': 44.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "StarCoder 15b",
      "column_label": "EvoEval pass@1",
      "raw_value": "23.5",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 12,
      "Creative": 11,
      "Subtle": 37,
      "Combine": 2,
      "Tool_Use": 44
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humaneval",
    "score": 0.421,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mixtral-Inst 8x7b' (Table 5 model ID 'mistralai/Mixtral-8x7B-Instruct-v0.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 42.1 (the bracketed HumanEval+ value 38.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mixtral-Inst 8x7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "42.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "evoeval",
    "score": 0.293,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mixtral-Inst 8x7b' (Table 5 model ID 'mistralai/Mixtral-8x7B-Instruct-v0.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 29.3: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 21.0, 'Creative': 18.0, 'Subtle': 41.0, 'Combine': 9.0, 'Tool_Use': 45.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mixtral-Inst 8x7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "29.3",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 21,
      "Creative": 18,
      "Subtle": 41,
      "Combine": 9,
      "Tool_Use": 45
    }
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humaneval",
    "score": 0.28,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mistral-Inst-v02 7b' (Table 5 model ID 'mistralai/Mistral-7B-Instruct-v0.2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 28.0 (the bracketed HumanEval+ value 23.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral-Inst-v02 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "28.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "evoeval",
    "score": 0.147,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mistral-Inst-v02 7b' (Table 5 model ID 'mistralai/Mistral-7B-Instruct-v0.2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 14.7: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 8.0, 'Creative': 16.0, 'Subtle': 25.0, 'Combine': 3.0, 'Tool_Use': 8.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral-Inst-v02 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "14.7",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 8,
      "Creative": 16,
      "Subtle": 25,
      "Combine": 3,
      "Tool_Use": 8
    }
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mistral-Inst 7b' (Table 5 model ID 'mistralai/Mistral-7B-Instruct-v0.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 28.7 (the bracketed HumanEval+ value 24.4 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral-Inst 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "evoeval",
    "score": 0.176,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Mistral-Inst 7b' (Table 5 model ID 'mistralai/Mistral-7B-Instruct-v0.1'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 17.6: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 6.0, 'Creative': 8.0, 'Subtle': 31.0, 'Combine': 3.0, 'Tool_Use': 29.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Mistral-Inst 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "17.6",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 6,
      "Creative": 8,
      "Subtle": 31,
      "Combine": 3,
      "Tool_Use": 29
    }
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "humaneval",
    "score": 0.713,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'OpenChat 7b' (Table 5 model ID 'openchat/openchat-3.5-0106'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 71.3 (the bracketed HumanEval+ value 66.5 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "OpenChat 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "71.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "evoeval",
    "score": 0.421,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'OpenChat 7b' (Table 5 model ID 'openchat/openchat-3.5-0106'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 42.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 33.0, 'Creative': 29.0, 'Subtle': 62.0, 'Combine': 14.0, 'Tool_Use': 43.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "OpenChat 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "42.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 33,
      "Creative": 29,
      "Subtle": 62,
      "Combine": 14,
      "Tool_Use": 43
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "humaneval",
    "score": 0.293,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'stable-code 3b' (Table 5 model ID 'stabilityai/stable-code-3b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 29.3 (the bracketed HumanEval+ value 25.6 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "stable-code 3b",
      "column_label": "HumanEval pass@1",
      "raw_value": "29.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "evoeval",
    "score": 0.207,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'stable-code 3b' (Table 5 model ID 'stabilityai/stable-code-3b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 20.7: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 10.0, 'Creative': 10.0, 'Subtle': 31.0, 'Combine': 3.0, 'Tool_Use': 41.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "stable-code 3b",
      "column_label": "EvoEval pass@1",
      "raw_value": "20.7",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 10,
      "Creative": 10,
      "Subtle": 31,
      "Combine": 3,
      "Tool_Use": 41
    }
  },
  {
    "model": "gemma-7b",
    "bench": "humaneval",
    "score": 0.28,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma-Inst 7b' (Table 5 model ID 'google/gemma-7b-it'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 28.0 (the bracketed HumanEval+ value 23.2 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma-Inst 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "28.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b",
    "bench": "evoeval",
    "score": 0.177,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma-Inst 7b' (Table 5 model ID 'google/gemma-7b-it'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 17.7: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 6.0, 'Creative': 10.0, 'Subtle': 29.0, 'Combine': 2.0, 'Tool_Use': 31.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma-Inst 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "17.7",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 6,
      "Creative": 10,
      "Subtle": 29,
      "Combine": 2,
      "Tool_Use": 31
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "humaneval",
    "score": 0.317,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma 7b' (Table 5 model ID 'google/gemma-7b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 31.7 (the bracketed HumanEval+ value 25.0 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "31.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "evoeval",
    "score": 0.23,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma 7b' (Table 5 model ID 'google/gemma-7b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 23.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 12.0, 'Creative': 13.0, 'Subtle': 40.0, 'Combine': 2.0, 'Tool_Use': 39.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "23.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 12,
      "Creative": 13,
      "Subtle": 40,
      "Combine": 2,
      "Tool_Use": 39
    }
  },
  {
    "model": "gemma-2b-base",
    "bench": "humaneval",
    "score": 0.22,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma 2b' (Table 5 model ID 'google/gemma-2b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 22.0 (the bracketed HumanEval+ value 17.1 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma 2b",
      "column_label": "HumanEval pass@1",
      "raw_value": "22.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b-base",
    "bench": "evoeval",
    "score": 0.128,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Gemma 2b' (Table 5 model ID 'google/gemma-2b'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 12.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 2.0, 'Creative': 6.0, 'Subtle': 24.0, 'Combine': 2.0, 'Tool_Use': 21.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Gemma 2b",
      "column_label": "EvoEval pass@1",
      "raw_value": "12.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 2,
      "Creative": 6,
      "Subtle": 24,
      "Combine": 2,
      "Tool_Use": 21
    }
  },
  {
    "model": "phi-2",
    "bench": "humaneval",
    "score": 0.5,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Phi-2 2.7b' (Table 5 model ID 'microsoft/phi-2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 50.0 (the bracketed HumanEval+ value 45.1 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Phi-2 2.7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "phi-2",
    "bench": "evoeval",
    "score": 0.318,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "base",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Phi-2 2.7b' (Table 5 model ID 'microsoft/phi-2'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 31.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 18.0, 'Creative': 23.0, 'Subtle': 49.0, 'Combine': 14.0, 'Tool_Use': 37.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Phi-2 2.7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "31.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 18,
      "Creative": 23,
      "Subtle": 49,
      "Combine": 14,
      "Tool_Use": 37
    }
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "humaneval",
    "score": 0.671,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 72b' (Table 5 model ID 'Qwen/Qwen1.5-72B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 67.1 (the bracketed HumanEval+ value 61.6 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 72b",
      "column_label": "HumanEval pass@1",
      "raw_value": "67.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "evoeval",
    "score": 0.42,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 72b' (Table 5 model ID 'Qwen/Qwen1.5-72B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 42.0: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 28.0, 'Creative': 28.0, 'Subtle': 61.0, 'Combine': 21.0, 'Tool_Use': 47.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 72b",
      "column_label": "EvoEval pass@1",
      "raw_value": "42.0",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 28,
      "Creative": 28,
      "Subtle": 61,
      "Combine": 21,
      "Tool_Use": 47
    }
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "humaneval",
    "score": 0.5,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 14b' (Table 5 model ID 'Qwen/Qwen1.5-14B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 50.0 (the bracketed HumanEval+ value 45.7 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 14b",
      "column_label": "HumanEval pass@1",
      "raw_value": "50.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "evoeval",
    "score": 0.338,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 14b' (Table 5 model ID 'Qwen/Qwen1.5-14B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 33.8: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 20.0, 'Creative': 23.0, 'Subtle': 48.0, 'Combine': 18.0, 'Tool_Use': 44.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 14b",
      "column_label": "EvoEval pass@1",
      "raw_value": "33.8",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 20,
      "Creative": 23,
      "Subtle": 48,
      "Combine": 18,
      "Tool_Use": 44
    }
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "humaneval",
    "score": 0.421,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 7b' (Table 5 model ID 'Qwen/Qwen1.5-7B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 42.1 (the bracketed HumanEval+ value 37.8 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 7b",
      "column_label": "HumanEval pass@1",
      "raw_value": "42.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "evoeval",
    "score": 0.255,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen-1.5 7b' (Table 5 model ID 'Qwen/Qwen1.5-7B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 25.5: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 16.0, 'Creative': 13.0, 'Subtle': 43.0, 'Combine': 7.0, 'Tool_Use': 32.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen-1.5 7b",
      "column_label": "EvoEval pass@1",
      "raw_value": "25.5",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 16,
      "Creative": 13,
      "Subtle": 43,
      "Combine": 7,
      "Tool_Use": 32
    }
  },
  {
    "model": "qwen-14b-chat",
    "bench": "humaneval",
    "score": 0.463,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen 14b' (Table 5 model ID 'Qwen/Qwen-14B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'HumanEval' pass@1 = 46.3 (the bracketed HumanEval+ value 43.9 is a re-scoring of the same generations and is not imported).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen 14b",
      "column_label": "HumanEval pass@1",
      "raw_value": "46.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen-14b-chat",
    "bench": "evoeval",
    "score": 0.321,
    "source": "https://arxiv.org/html/2403.19114v1",
    "source_id": "paper-2403.19114v1",
    "by": "independent",
    "date": "2024-03-28",
    "mode": "instruct",
    "note": "EvoEval paper (arXiv:2403.19114v1, Xia, Deng, Dunn, Zhang; UIUC) Table 2, row 'Qwen 14b' (Table 5 model ID 'Qwen/Qwen-14B-Chat'). Greedy decoding pass@1; 'We evaluate 51 popular state-of-the-art LLMs' (Section 4), open models run locally with the released weights and closed models through the named API endpoint (Table 5). Column 'EvoEval' pass@1 = 32.1: the paper's printed overall score, which equals the mean of the HumanEval column and the five evolved benchmarks Difficult/Creative/Subtle/Combine/Tool_Use (components as printed: {'Difficult': 23.0, 'Creative': 20.0, 'Subtle': 45.0, 'Combine': 13.0, 'Tool_Use': 45.0}).",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Qwen 14b",
      "column_label": "EvoEval pass@1",
      "raw_value": "32.1",
      "scale_divisor": 100
    },
    "components": {
      "Difficult": 23,
      "Creative": 20,
      "Subtle": 45,
      "Combine": 13,
      "Tool_Use": 45
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humaneval",
    "score": 0.213,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-3B', column 'HumanEval' = 21.3. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-3B",
      "column_label": "HumanEval",
      "raw_value": "21.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "mbpp",
    "score": 0.426,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-3B', column 'MBPP' = 42.6. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-3B",
      "column_label": "MBPP",
      "raw_value": "42.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-1.3B', column 'HumanEval' = 28.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-1.3B",
      "column_label": "HumanEval",
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "mbpp",
    "score": 0.554,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-1.3B', column 'MBPP' = 55.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-1.3B",
      "column_label": "MBPP",
      "raw_value": "55.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StableCode-3B', column 'HumanEval' = 28.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StableCode-3B",
      "column_label": "HumanEval",
      "raw_value": "28.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "mbpp",
    "score": 0.531,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StableCode-3B', column 'MBPP' = 53.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StableCode-3B",
      "column_label": "MBPP",
      "raw_value": "53.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "humaneval",
    "score": 0.317,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-3B', column 'HumanEval' = 31.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-3B",
      "column_label": "HumanEval",
      "raw_value": "31.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "mbpp",
    "score": 0.574,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-3B', column 'MBPP' = 57.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-3B",
      "column_label": "MBPP",
      "raw_value": "57.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humaneval",
    "score": 0.305,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-7B', column 'HumanEval' = 30.5. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-7B",
      "column_label": "HumanEval",
      "raw_value": "30.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "mbpp",
    "score": 0.474,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-7B', column 'MBPP' = 47.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-7B",
      "column_label": "MBPP",
      "raw_value": "47.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "humaneval",
    "score": 0.335,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-7B', column 'HumanEval' = 33.5. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-7B",
      "column_label": "HumanEval",
      "raw_value": "33.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "mbpp",
    "score": 0.521,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-7B', column 'MBPP' = 52.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-7B",
      "column_label": "MBPP",
      "raw_value": "52.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "humaneval",
    "score": 0.476,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-6.7B', column 'HumanEval' = 47.6. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-6.7B",
      "column_label": "HumanEval",
      "raw_value": "47.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "mbpp",
    "score": 0.702,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-6.7B', column 'MBPP' = 70.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-6.7B",
      "column_label": "MBPP",
      "raw_value": "70.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "humaneval",
    "score": 0.354,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-7B', column 'HumanEval' = 35.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-7B",
      "column_label": "HumanEval",
      "raw_value": "35.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "mbpp",
    "score": 0.544,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-7B', column 'MBPP' = 54.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-7B",
      "column_label": "MBPP",
      "raw_value": "54.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "humaneval",
    "score": 0.293,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-15B', column 'HumanEval' = 29.3. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-15B",
      "column_label": "HumanEval",
      "raw_value": "29.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "mbpp",
    "score": 0.506,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoderBase-15B', column 'MBPP' = 50.6. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoderBase-15B",
      "column_label": "MBPP",
      "raw_value": "50.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "humaneval",
    "score": 0.378,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-13B', column 'HumanEval' = 37.8. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-13B",
      "column_label": "HumanEval",
      "raw_value": "37.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "mbpp",
    "score": 0.624,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-13B', column 'MBPP' = 62.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-13B",
      "column_label": "MBPP",
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "humaneval",
    "score": 0.463,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-15B', column 'HumanEval' = 46.3. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-15B",
      "column_label": "HumanEval",
      "raw_value": "46.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "mbpp",
    "score": 0.662,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'StarCoder2-15B', column 'MBPP' = 66.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "StarCoder2-15B",
      "column_label": "MBPP",
      "raw_value": "66.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "humaneval",
    "score": 0.482,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-34B', column 'HumanEval' = 48.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-34B",
      "column_label": "HumanEval",
      "raw_value": "48.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "mbpp",
    "score": 0.654,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'CodeLlama-34B', column 'MBPP' = 65.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "CodeLlama-34B",
      "column_label": "MBPP",
      "raw_value": "65.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "humaneval",
    "score": 0.543,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-33B', column 'HumanEval' = 54.3. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus framework, all 164 problems. HumanEval+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-33B",
      "column_label": "HumanEval",
      "raw_value": "54.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "mbpp",
    "score": 0.732,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 9 'Pass@1 on HumanEval(+) and MBPP(+)', row 'DeepSeekCoder-33B', column 'MBPP' = 73.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Greedy decoding, pass@1, EvalPlus's 399-problem sanitized MBPP subset. MBPP+ (same generations re-scored) not imported.",
    "evidence": {
      "kind": "table",
      "table": "Table 9",
      "row_label": "DeepSeekCoder-33B",
      "column_label": "MBPP",
      "raw_value": "73.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "ds-1000",
    "score": 0.142,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoderBase-3B', column 'Overall' = 14.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoderBase-3B",
      "column_label": "Overall",
      "raw_value": "14.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "ds-1000",
    "score": 0.227,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StableCode-3B', column 'Overall' = 22.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StableCode-3B",
      "column_label": "Overall",
      "raw_value": "22.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "ds-1000",
    "score": 0.162,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'DeepSeekCoder-1.3B', column 'Overall' = 16.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "DeepSeekCoder-1.3B",
      "column_label": "Overall",
      "raw_value": "16.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "ds-1000",
    "score": 0.25,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoder2-3B', column 'Overall' = 25.0. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoder2-3B",
      "column_label": "Overall",
      "raw_value": "25.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "ds-1000",
    "score": 0.191,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoderBase-7B', column 'Overall' = 19.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoderBase-7B",
      "column_label": "Overall",
      "raw_value": "19.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "ds-1000",
    "score": 0.289,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'DeepSeekCoder-6.7B', column 'Overall' = 28.9. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "DeepSeekCoder-6.7B",
      "column_label": "Overall",
      "raw_value": "28.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "ds-1000",
    "score": 0.215,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'CodeLlama-7B', column 'Overall' = 21.5. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "CodeLlama-7B",
      "column_label": "Overall",
      "raw_value": "21.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "ds-1000",
    "score": 0.278,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoder2-7B', column 'Overall' = 27.8. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoder2-7B",
      "column_label": "Overall",
      "raw_value": "27.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "ds-1000",
    "score": 0.238,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoderBase-15B', column 'Overall' = 23.8. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoderBase-15B",
      "column_label": "Overall",
      "raw_value": "23.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "ds-1000",
    "score": 0.251,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'CodeLlama-13B', column 'Overall' = 25.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "CodeLlama-13B",
      "column_label": "Overall",
      "raw_value": "25.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "ds-1000",
    "score": 0.338,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'StarCoder2-15B', column 'Overall' = 33.8. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "StarCoder2-15B",
      "column_label": "Overall",
      "raw_value": "33.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "ds-1000",
    "score": 0.402,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'DeepSeekCoder-33B', column 'Overall' = 40.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "DeepSeekCoder-33B",
      "column_label": "Overall",
      "raw_value": "40.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "ds-1000",
    "score": 0.343,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 11 'Performance of open-access models on DS-1000', row 'CodeLlama-34B', column 'Overall' = 34.3. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. Completion format, temperature 0.2, top-p 0.95, mean pass@1 over 40 samples, all 1,000 problems.",
    "evidence": {
      "kind": "table",
      "table": "Table 11",
      "row_label": "CodeLlama-34B",
      "column_label": "Overall",
      "raw_value": "34.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "gsm8k",
    "score": 0.08,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoderBase-3B', column 'GSM8K (PAL)' = 8.0. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-3B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "8.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "gsm8k",
    "score": 0.126,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'DeepSeekCoder-1.3B', column 'GSM8K (PAL)' = 12.6. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "DeepSeekCoder-1.3B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "12.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "gsm8k",
    "score": 0.397,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StableCode-3B', column 'GSM8K (PAL)' = 39.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StableCode-3B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "39.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "gsm8k",
    "score": 0.277,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoder2-3B', column 'GSM8K (PAL)' = 27.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-3B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "27.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "gsm8k",
    "score": 0.149,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoderBase-7B', column 'GSM8K (PAL)' = 14.9. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-7B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "14.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "gsm8k",
    "score": 0.419,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'DeepSeekCoder-6.7B', column 'GSM8K (PAL)' = 41.9. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "DeepSeekCoder-6.7B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "41.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "gsm8k",
    "score": 0.27,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'CodeLlama-7B', column 'GSM8K (PAL)' = 27.0. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-7B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "27.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "gsm8k",
    "score": 0.404,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoder2-7B', column 'GSM8K (PAL)' = 40.4. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-7B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "40.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "gsm8k",
    "score": 0.215,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoderBase-15B', column 'GSM8K (PAL)' = 21.5. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-15B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "21.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "gsm8k",
    "score": 0.381,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'CodeLlama-13B', column 'GSM8K (PAL)' = 38.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-13B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "38.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "gsm8k",
    "score": 0.651,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'StarCoder2-15B', column 'GSM8K (PAL)' = 65.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-15B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "65.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "gsm8k",
    "score": 0.542,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'CodeLlama-34B', column 'GSM8K (PAL)' = 54.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-34B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "54.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "gsm8k",
    "score": 0.587,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "independent",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 14 '8-shot accuracy on the GSM8K math-reasoning benchmark', row 'DeepSeekCoder-33B', column 'GSM8K (PAL)' = 58.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. PAL (program-aided) prompting, 8-shot, greedy decoding.",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "DeepSeekCoder-33B",
      "column_label": "GSM8K (PAL)",
      "raw_value": "58.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "cruxeval-i",
    "score": 0.327,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-3B', column 'CRUXEval-I Pass@1' = 32.7. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-3B",
      "column_label": "CRUXEval-I Pass@1",
      "raw_value": "32.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "cruxeval-o",
    "score": 0.342,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-3B', column 'CRUXEval-O Pass@1' = 34.2. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-3B",
      "column_label": "CRUXEval-O Pass@1",
      "raw_value": "34.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "cruxeval-i",
    "score": 0.346,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-7B', column 'CRUXEval-I Pass@1' = 34.6. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-7B",
      "column_label": "CRUXEval-I Pass@1",
      "raw_value": "34.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "cruxeval-o",
    "score": 0.36,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-7B', column 'CRUXEval-O Pass@1' = 36.0. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-7B",
      "column_label": "CRUXEval-O Pass@1",
      "raw_value": "36.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "cruxeval-i",
    "score": 0.481,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-15B', column 'CRUXEval-I Pass@1' = 48.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-15B",
      "column_label": "CRUXEval-I Pass@1",
      "raw_value": "48.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "cruxeval-o",
    "score": 0.471,
    "source": "https://arxiv.org/html/2402.19173v1",
    "source_id": "paper-2402.19173v1",
    "by": "lab",
    "date": "2024-02-29",
    "mode": "base",
    "note": "StarCoder 2 paper (arXiv:2402.19173v1, BigCode) Table 15 'Accuracy on the CRUXEval benchmark', row 'StarCoder2-15B', column 'CRUXEval-O Pass@1' = 47.1. Section 7: 'We evaluate the StarCoder2 models ... and compare them to ... StableCode, Code Llama, DeepSeekCoder, and original StarCoder' (base models only); tables that reuse other papers' numbers say so in their captions (CanItEdit, FIM) and are not used. CRUXEval direct prediction, pass@1 (CRUXEval protocol).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-15B",
      "column_label": "CRUXEval-O Pass@1",
      "raw_value": "47.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humanevalsynthesize",
    "score": 0.218,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoderBase-3B' (Completion prompt), column 'Avg.' = 21.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoderBase-3B",
      "column_label": "Avg.",
      "raw_value": "21.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "humanevalsynthesize",
    "score": 0.278,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StableCode-3B' (Completion prompt), column 'Avg.' = 27.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StableCode-3B",
      "column_label": "Avg.",
      "raw_value": "27.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "humanevalsynthesize",
    "score": 0.317,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoder2-3B' (Completion prompt), column 'Avg.' = 31.7 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoder2-3B",
      "column_label": "Avg.",
      "raw_value": "31.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "humanevalsynthesize",
    "score": 0.304,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeGemma-2B' (Completion prompt), column 'Avg.' = 30.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeGemma-2B",
      "column_label": "Avg.",
      "raw_value": "30.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "humanevalsynthesize",
    "score": 0.331,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-3B-Code-Base' (Completion prompt), column 'Avg.' = 33.1 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-3B-Code-Base",
      "column_label": "Avg.",
      "raw_value": "33.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humanevalsynthesize",
    "score": 0.267,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoderBase-7B' (Completion prompt), column 'Avg.' = 26.7 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoderBase-7B",
      "column_label": "Avg.",
      "raw_value": "26.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "humanevalsynthesize",
    "score": 0.312,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-7B' (Completion prompt), column 'Avg.' = 31.2 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-7B",
      "column_label": "Avg.",
      "raw_value": "31.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "humanevalsynthesize",
    "score": 0.374,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoder2-7B' (Completion prompt), column 'Avg.' = 37.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoder2-7B",
      "column_label": "Avg.",
      "raw_value": "37.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "humanevalsynthesize",
    "score": 0.415,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeGemma-7B' (Completion prompt), column 'Avg.' = 41.5 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeGemma-7B",
      "column_label": "Avg.",
      "raw_value": "41.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "humanevalsynthesize",
    "score": 0.435,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-8B-Code-Base' (Completion prompt), column 'Avg.' = 43.5 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "Avg.",
      "raw_value": "43.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "humanevalsynthesize",
    "score": 0.319,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoderBase-15B' (Completion prompt), column 'Avg.' = 31.9 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoderBase-15B",
      "column_label": "Avg.",
      "raw_value": "31.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "humanevalsynthesize",
    "score": 0.378,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-13B' (Completion prompt), column 'Avg.' = 37.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-13B",
      "column_label": "Avg.",
      "raw_value": "37.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "humanevalsynthesize",
    "score": 0.444,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'StarCoder2-15B' (Completion prompt), column 'Avg.' = 44.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "StarCoder2-15B",
      "column_label": "Avg.",
      "raw_value": "44.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "humanevalsynthesize",
    "score": 0.443,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-20B-Code-Base' (Completion prompt), column 'Avg.' = 44.3 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-20B-Code-Base",
      "column_label": "Avg.",
      "raw_value": "44.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "humanevalsynthesize",
    "score": 0.433,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-34B' (Completion prompt), column 'Avg.' = 43.3 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-34B",
      "column_label": "Avg.",
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "humanevalsynthesize",
    "score": 0.491,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-34B-Code-Base' (Completion prompt), column 'Avg.' = 49.1 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-34B-Code-Base",
      "column_label": "Avg.",
      "raw_value": "49.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b",
    "bench": "humanevalsynthesize",
    "score": 0.528,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-70B' (Completion prompt), column 'Avg.' = 52.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-70B",
      "column_label": "Avg.",
      "raw_value": "52.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b-base",
    "bench": "humanevalsynthesize",
    "score": 0.172,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Gemma-2B' (Completion prompt), column 'Avg.' = 17.2 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma-2B",
      "column_label": "Avg.",
      "raw_value": "17.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "humanevalsynthesize",
    "score": 0.344,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Gemma-7B' (Completion prompt), column 'Avg.' = 34.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma-7B",
      "column_label": "Avg.",
      "raw_value": "34.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "humanevalsynthesize",
    "score": 0.433,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Mixtral-8x7B-v0.1' (Completion prompt), column 'Avg.' = 43.3 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Mixtral-8x7B-v0.1",
      "column_label": "Avg.",
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humanevalsynthesize",
    "score": 0.416,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeGemma-7B-IT' (Instruction prompt), column 'Avg.' = 41.6 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeGemma-7B-IT",
      "column_label": "Avg.",
      "raw_value": "41.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "humanevalsynthesize",
    "score": 0.379,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-7B-Instruct' (Instruction prompt), column 'Avg.' = 37.9 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-7B-Instruct",
      "column_label": "Avg.",
      "raw_value": "37.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "humanevalsynthesize",
    "score": 0.394,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-13B-Instruct' (Instruction prompt), column 'Avg.' = 39.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-13B-Instruct",
      "column_label": "Avg.",
      "raw_value": "39.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "humanevalsynthesize",
    "score": 0.413,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-34B-Instruct' (Instruction prompt), column 'Avg.' = 41.3 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-34B-Instruct",
      "column_label": "Avg.",
      "raw_value": "41.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humanevalsynthesize",
    "score": 0.589,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'CodeLlama-70B-Instruct' (Instruction prompt), column 'Avg.' = 58.9 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "CodeLlama-70B-Instruct",
      "column_label": "Avg.",
      "raw_value": "58.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "humanevalsynthesize",
    "score": 0.396,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-3b-Code-Instruct' (Instruction prompt), column 'Avg.' = 39.6 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-3b-Code-Instruct",
      "column_label": "Avg.",
      "raw_value": "39.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-instruct-4k",
    "bench": "humanevalsynthesize",
    "score": 0.496,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-8b-Code-Instruct' (Instruction prompt), column 'Avg.' = 49.6 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-8b-Code-Instruct",
      "column_label": "Avg.",
      "raw_value": "49.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "humanevalsynthesize",
    "score": 0.505,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-20B-Code-Instruct' (Instruction prompt), column 'Avg.' = 50.5 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-20B-Code-Instruct",
      "column_label": "Avg.",
      "raw_value": "50.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "humanevalsynthesize",
    "score": 0.548,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Granite-34B-Code-Instruct' (Instruction prompt), column 'Avg.' = 54.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Granite-34B-Code-Instruct",
      "column_label": "Avg.",
      "raw_value": "54.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b",
    "bench": "humanevalsynthesize",
    "score": 0.115,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Gemma-2B-IT' (Instruction prompt), column 'Avg.' = 11.5 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma-2B-IT",
      "column_label": "Avg.",
      "raw_value": "11.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b",
    "bench": "humanevalsynthesize",
    "score": 0.198,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Gemma-7B-IT' (Instruction prompt), column 'Avg.' = 19.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma-7B-IT",
      "column_label": "Avg.",
      "raw_value": "19.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humanevalsynthesize",
    "score": 0.307,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Mistral-7B-Instruct-v0.2' (Instruction prompt), column 'Avg.' = 30.7 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Mistral-7B-Instruct-v0.2",
      "column_label": "Avg.",
      "raw_value": "30.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humanevalsynthesize",
    "score": 0.484,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Mixtral-8x7B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 48.4 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Mixtral-8x7B-Instruct-v0.1",
      "column_label": "Avg.",
      "raw_value": "48.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "humanevalsynthesize",
    "score": 0.648,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Mixtral-8x22B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 64.8 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Mixtral-8x22B-Instruct-v0.1",
      "column_label": "Avg.",
      "raw_value": "64.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-8b",
    "bench": "humanevalsynthesize",
    "score": 0.369,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Llama-3-8B-Instruct' (Instruction prompt), column 'Avg.' = 36.9 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama-3-8B-Instruct",
      "column_label": "Avg.",
      "raw_value": "36.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "llama-3-70b",
    "bench": "humanevalsynthesize",
    "score": 0.655,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 3 'Pass@1 performance on HumanEvalSynthesize', row 'Llama-3-70B-Instruct' (Instruction prompt), column 'Avg.' = 65.5 (mean of Python/JavaScript/Java/Go/C++/Rust as printed). Greedy decoding; completion format for base models, instruction template for instruct models. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Llama-3-70B-Instruct",
      "column_label": "Avg.",
      "raw_value": "65.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "mbpp",
    "score": 0.294,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoderBase-3B', column 'MBPP' = 29.4. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoderBase-3B",
      "column_label": "MBPP",
      "raw_value": "29.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "mbpp",
    "score": 0.348,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StableCode-3B', column 'MBPP' = 34.8. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StableCode-3B",
      "column_label": "MBPP",
      "raw_value": "34.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "mbpp",
    "score": 0.424,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoder2-3B', column 'MBPP' = 42.4. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoder2-3B",
      "column_label": "MBPP",
      "raw_value": "42.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "mbpp",
    "score": 0.304,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'CodeGemma-2B', column 'MBPP' = 30.4. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "CodeGemma-2B",
      "column_label": "MBPP",
      "raw_value": "30.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "mbpp",
    "score": 0.36,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'Granite-3B-Code-Base', column 'MBPP' = 36.0. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Granite-3B-Code-Base",
      "column_label": "MBPP",
      "raw_value": "36.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "mbpp",
    "score": 0.348,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoderBase-7B', column 'MBPP' = 34.8. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoderBase-7B",
      "column_label": "MBPP",
      "raw_value": "34.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "mbpp",
    "score": 0.39,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'CodeLlama-7B', column 'MBPP' = 39.0. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "CodeLlama-7B",
      "column_label": "MBPP",
      "raw_value": "39.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "mbpp",
    "score": 0.454,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoder2-7B', column 'MBPP' = 45.4. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoder2-7B",
      "column_label": "MBPP",
      "raw_value": "45.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "mbpp",
    "score": 0.53,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'CodeGemma-7B', column 'MBPP' = 53.0. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "CodeGemma-7B",
      "column_label": "MBPP",
      "raw_value": "53.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "mbpp",
    "score": 0.422,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'Granite-8B-Code-Base', column 'MBPP' = 42.2. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "MBPP",
      "raw_value": "42.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "mbpp",
    "score": 0.374,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoderBase-15B', column 'MBPP' = 37.4. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoderBase-15B",
      "column_label": "MBPP",
      "raw_value": "37.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "mbpp",
    "score": 0.306,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'CodeLlama-13B', column 'MBPP' = 30.6. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "CodeLlama-13B",
      "column_label": "MBPP",
      "raw_value": "30.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "mbpp",
    "score": 0.512,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'StarCoder2-15B', column 'MBPP' = 51.2. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "StarCoder2-15B",
      "column_label": "MBPP",
      "raw_value": "51.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "mbpp",
    "score": 0.438,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'Granite-20B-Code-Base', column 'MBPP' = 43.8. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Granite-20B-Code-Base",
      "column_label": "MBPP",
      "raw_value": "43.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "mbpp",
    "score": 0.486,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'CodeLlama-34B', column 'MBPP' = 48.6. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "CodeLlama-34B",
      "column_label": "MBPP",
      "raw_value": "48.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "mbpp",
    "score": 0.472,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 5 'Pass@1 on MBPP and MBPP+', row 'Granite-34B-Code-Base', column 'MBPP' = 47.2. Zero-shot, greedy decoding. MBPP+ (same generations re-scored) not imported. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 5",
      "row_label": "Granite-34B-Code-Base",
      "column_label": "MBPP",
      "raw_value": "47.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "cruxeval-i",
    "score": 0.275,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-3B', column 'CRUXEval-I Pass1' = 27.5. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-3B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "27.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-3b",
    "bench": "cruxeval-o",
    "score": 0.27,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-3B', column 'CRUXEval-O Pass1' = 27.0. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-3B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "27.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "cruxeval-i",
    "score": 0.335,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StableCode-3B', column 'CRUXEval-I Pass1' = 33.5. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StableCode-3B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "33.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "stable-code-3b",
    "bench": "cruxeval-o",
    "score": 0.321,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StableCode-3B', column 'CRUXEval-O Pass1' = 32.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StableCode-3B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "32.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "cruxeval-i",
    "score": 0.321,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-3B', column 'CRUXEval-I Pass1' = 32.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-3B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "32.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-3b",
    "bench": "cruxeval-o",
    "score": 0.34,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-3B', column 'CRUXEval-O Pass1' = 34.0. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-3B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "34.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "cruxeval-i",
    "score": 0.296,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeGemma-2B', column 'CRUXEval-I Pass1' = 29.6. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeGemma-2B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "29.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "cruxeval-o",
    "score": 0.321,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeGemma-2B', column 'CRUXEval-O Pass1' = 32.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeGemma-2B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "32.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "cruxeval-i",
    "score": 0.306,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-3B-Code-Base', column 'CRUXEval-I Pass1' = 30.6. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-3B-Code-Base",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "30.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "cruxeval-o",
    "score": 0.314,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-3B-Code-Base', column 'CRUXEval-O Pass1' = 31.4. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-3B-Code-Base",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "31.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "cruxeval-i",
    "score": 0.298,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-7B', column 'CRUXEval-I Pass1' = 29.8. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-7B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "29.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "cruxeval-o",
    "score": 0.321,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-7B', column 'CRUXEval-O Pass1' = 32.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-7B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "32.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "cruxeval-i",
    "score": 0.362,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-7B', column 'CRUXEval-I Pass1' = 36.2. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-7B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "36.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "cruxeval-o",
    "score": 0.34,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-7B', column 'CRUXEval-O Pass1' = 34.0. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-7B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "34.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "cruxeval-i",
    "score": 0.342,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-7B', column 'CRUXEval-I Pass1' = 34.2. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-7B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "34.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "cruxeval-o",
    "score": 0.358,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-7B', column 'CRUXEval-O Pass1' = 35.8. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-7B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "35.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "cruxeval-i",
    "score": 0.426,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeGemma-7B', column 'CRUXEval-I Pass1' = 42.6. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeGemma-7B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "42.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "cruxeval-o",
    "score": 0.439,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeGemma-7B', column 'CRUXEval-O Pass1' = 43.9. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeGemma-7B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "43.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "cruxeval-i",
    "score": 0.36,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-8B-Code-Base', column 'CRUXEval-I Pass1' = 36.0. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "36.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "cruxeval-o",
    "score": 0.361,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-8B-Code-Base', column 'CRUXEval-O Pass1' = 36.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "36.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "cruxeval-i",
    "score": 0.31,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-15B', column 'CRUXEval-I Pass1' = 31.0. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-15B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "31.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase",
    "bench": "cruxeval-o",
    "score": 0.344,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoderBase-15B', column 'CRUXEval-O Pass1' = 34.4. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoderBase-15B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "34.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "cruxeval-i",
    "score": 0.422,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-13B', column 'CRUXEval-I Pass1' = 42.2. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-13B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "42.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "cruxeval-o",
    "score": 0.399,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-13B', column 'CRUXEval-O Pass1' = 39.9. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-13B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "39.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "cruxeval-i",
    "score": 0.474,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-15B', column 'CRUXEval-I Pass1' = 47.4. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-15B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "47.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-15b",
    "bench": "cruxeval-o",
    "score": 0.467,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'StarCoder2-15B', column 'CRUXEval-O Pass1' = 46.7. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "StarCoder2-15B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "46.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "cruxeval-i",
    "score": 0.391,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-20B-Code-Base', column 'CRUXEval-I Pass1' = 39.1. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-20B-Code-Base",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "39.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "cruxeval-o",
    "score": 0.375,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-20B-Code-Base', column 'CRUXEval-O Pass1' = 37.5. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-20B-Code-Base",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "37.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "cruxeval-i",
    "score": 0.478,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-34B', column 'CRUXEval-I Pass1' = 47.8. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-34B",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "47.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "cruxeval-o",
    "score": 0.425,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'CodeLlama-34B', column 'CRUXEval-O Pass1' = 42.5. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "CodeLlama-34B",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "42.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "cruxeval-i",
    "score": 0.433,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-34B-Code-Base', column 'CRUXEval-I Pass1' = 43.3. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-34B-Code-Base",
      "column_label": "CRUXEval-I Pass1",
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "cruxeval-o",
    "score": 0.448,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 14 'Performance on the CRUXEval benchmark', row 'Granite-34B-Code-Base', column 'CRUXEval-O Pass1' = 44.8. Temperature 0.2, 10 samples, pass@1 (CRUXEval protocol); values differ from the CRUXEval authors' own published results, consistent with IBM's own runs. 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 14",
      "row_label": "Granite-34B-Code-Base",
      "column_label": "CRUXEval-O Pass1",
      "raw_value": "44.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "math",
    "score": 0.024,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'StarCoderBase-7B', column 'MATH' = 2.4 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoderBase-7B",
      "column_label": "MATH",
      "raw_value": "2.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoderbase-7b",
    "bench": "gsm8k",
    "score": 0.038,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'StarCoderBase-7B', column 'GSM8K' = 3.8 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoderBase-7B",
      "column_label": "GSM8K",
      "raw_value": "3.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "math",
    "score": 0.041,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'CodeLlama-7B', column 'MATH' = 4.1 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "CodeLlama-7B",
      "column_label": "MATH",
      "raw_value": "4.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "gsm8k",
    "score": 0.119,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'CodeLlama-7B', column 'GSM8K' = 11.9 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "CodeLlama-7B",
      "column_label": "GSM8K",
      "raw_value": "11.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "math",
    "score": 0.104,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'StarCoder2-7B', column 'MATH' = 10.4 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-7B",
      "column_label": "MATH",
      "raw_value": "10.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "starcoder2-7b",
    "bench": "gsm8k",
    "score": 0.272,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'StarCoder2-7B', column 'GSM8K' = 27.2 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "StarCoder2-7B",
      "column_label": "GSM8K",
      "raw_value": "27.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "math",
    "score": 0.218,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'CodeGemma-7B', column 'MATH' = 21.8 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "CodeGemma-7B",
      "column_label": "MATH",
      "raw_value": "21.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "gsm8k",
    "score": 0.49,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'CodeGemma-7B', column 'GSM8K' = 49.0 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "CodeGemma-7B",
      "column_label": "GSM8K",
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "math",
    "score": 0.214,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'Granite-8B-Code-Base', column 'MATH' = 21.4 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "MATH",
      "raw_value": "21.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "gsm8k",
    "score": 0.619,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "lab",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'Granite-8B-Code-Base', column 'GSM8K' = 61.9 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Granite-8B-Code-Base",
      "column_label": "GSM8K",
      "raw_value": "61.9",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "math",
    "score": 0.241,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'Gemma-7B', column 'MATH' = 24.1 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Gemma-7B",
      "column_label": "MATH",
      "raw_value": "24.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "gsm8k",
    "score": 0.533,
    "source": "https://arxiv.org/html/2405.04324v1",
    "source_id": "paper-2405.04324v1",
    "by": "independent",
    "date": "2024-05-07",
    "mode": "base",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 15 'Performance on 4 chain-of-thought math tasks and 2 tool-aided math tasks', row 'Gemma-7B', column 'GSM8K' = 53.3 (chain-of-thought, few-shot as in Azerbayev et al. / Llemma setup). 'For all the benchmarks, we evaluate the baseline models (including ours) using the same script and environment for fair comparison' (Section 6).",
    "evidence": {
      "kind": "table",
      "table": "Table 15",
      "row_label": "Gemma-7B",
      "column_label": "GSM8K",
      "raw_value": "53.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "humaneval",
    "score": 0.335,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 7B', column 'HumanEval pass@1' = 33.5%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 7B",
      "column_label": "HumanEval pass@1",
      "raw_value": "33.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b",
    "bench": "mbpp",
    "score": 0.414,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 7B', column 'MBPP pass@1' = 41.4%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 7B",
      "column_label": "MBPP pass@1",
      "raw_value": "41.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "humaneval",
    "score": 0.36,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 13B', column 'HumanEval pass@1' = 36.0%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 13B",
      "column_label": "HumanEval pass@1",
      "raw_value": "36.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b",
    "bench": "mbpp",
    "score": 0.47,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 13B', column 'MBPP pass@1' = 47.0%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 13B",
      "column_label": "MBPP pass@1",
      "raw_value": "47.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "humaneval",
    "score": 0.488,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 34B', column 'HumanEval pass@1' = 48.8%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 34B",
      "column_label": "HumanEval pass@1",
      "raw_value": "48.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b",
    "bench": "mbpp",
    "score": 0.55,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 34B', column 'MBPP pass@1' = 55.0%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 34B",
      "column_label": "MBPP pass@1",
      "raw_value": "55.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b",
    "bench": "humaneval",
    "score": 0.53,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 70B', column 'HumanEval pass@1' = 53.0%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 70B",
      "column_label": "HumanEval pass@1",
      "raw_value": "53.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b",
    "bench": "mbpp",
    "score": 0.624,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama 70B', column 'MBPP pass@1' = 62.4%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama 70B",
      "column_label": "MBPP pass@1",
      "raw_value": "62.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "humaneval",
    "score": 0.34756,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 7B', column 'HumanEval pass@1' = 34.756%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 7B",
      "column_label": "HumanEval pass@1",
      "raw_value": "34.756",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "mbpp",
    "score": 0.444,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 7B', column 'MBPP pass@1' = 44.400%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 7B",
      "column_label": "MBPP pass@1",
      "raw_value": "44.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "humaneval",
    "score": 0.42683,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 13B', column 'HumanEval pass@1' = 42.683%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 13B",
      "column_label": "HumanEval pass@1",
      "raw_value": "42.683",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "mbpp",
    "score": 0.494,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 13B', column 'MBPP pass@1' = 49.400%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 13B",
      "column_label": "MBPP pass@1",
      "raw_value": "49.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "humaneval",
    "score": 0.415,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 34B', column 'HumanEval pass@1' = 41.5%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 34B",
      "column_label": "HumanEval pass@1",
      "raw_value": "41.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "mbpp",
    "score": 0.57,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 34B', column 'MBPP pass@1' = 57.0%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 34B",
      "column_label": "MBPP pass@1",
      "raw_value": "57.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humaneval",
    "score": 0.678,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 70B', column 'HumanEval pass@1' = 67.8%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 70B",
      "column_label": "HumanEval pass@1",
      "raw_value": "67.8",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "mbpp",
    "score": 0.622,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "instruct",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Instruct 70B', column 'MBPP pass@1' = 62.2%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Instruct 70B",
      "column_label": "MBPP pass@1",
      "raw_value": "62.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-python",
    "bench": "humaneval",
    "score": 0.384,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 7B', column 'HumanEval pass@1' = 38.4%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 7B",
      "column_label": "HumanEval pass@1",
      "raw_value": "38.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-7b-python",
    "bench": "mbpp",
    "score": 0.476,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 7B', column 'MBPP pass@1' = 47.6%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 7B",
      "column_label": "MBPP pass@1",
      "raw_value": "47.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-python",
    "bench": "humaneval",
    "score": 0.433,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 13B', column 'HumanEval pass@1' = 43.3%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 13B",
      "column_label": "HumanEval pass@1",
      "raw_value": "43.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-13b-python",
    "bench": "mbpp",
    "score": 0.49,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 13B', column 'MBPP pass@1' = 49.0%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 13B",
      "column_label": "MBPP pass@1",
      "raw_value": "49.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-python",
    "bench": "humaneval",
    "score": 0.537,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 34B', column 'HumanEval pass@1' = 53.7%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 34B",
      "column_label": "HumanEval pass@1",
      "raw_value": "53.7",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-34b-python",
    "bench": "mbpp",
    "score": 0.562,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 34B', column 'MBPP pass@1' = 56.2%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 34B",
      "column_label": "MBPP pass@1",
      "raw_value": "56.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-python",
    "bench": "humaneval",
    "score": 0.573,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 70B', column 'HumanEval pass@1' = 57.3%; zero-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 70B",
      "column_label": "HumanEval pass@1",
      "raw_value": "57.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "codellama-70b-python",
    "bench": "mbpp",
    "score": 0.656,
    "source": "https://arxiv.org/html/2308.12950v3",
    "source_id": "paper-2308.12950v3",
    "by": "lab",
    "date": "2024-01-31",
    "mode": "base",
    "note": "Code Llama paper (arXiv:2308.12950v3, Meta) Table 2 'Code Llama pass@ scores on HumanEval and MBPP', row 'Code Llama - Python 70B', column 'MBPP pass@1' = 65.6%; 3-shot, greedy decoding ('The pass@1 scores of our models are computed with greedy decoding'). Meta's own model, by: lab. Rows for other models in this table are copied from other papers (caption) and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 2",
      "row_label": "Code Llama - Python 70B",
      "column_label": "MBPP pass@1",
      "raw_value": "65.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "humaneval",
    "score": 0.311,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '2B-PT', column 'HumanEval' = 31.1%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "2B-PT",
      "column_label": "HumanEval",
      "raw_value": "31.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-2b",
    "bench": "mbpp",
    "score": 0.436,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '2B-PT', column 'MBPP' = 43.6%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "2B-PT",
      "column_label": "MBPP",
      "raw_value": "43.6",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b-base",
    "bench": "humaneval",
    "score": 0.22,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row 'Gemma 2B PT', column 'HumanEval' = 22.0%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma 2B PT",
      "column_label": "HumanEval",
      "raw_value": "22.0",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-2b-base",
    "bench": "mbpp",
    "score": 0.292,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row 'Gemma 2B PT', column 'MBPP' = 29.2%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma 2B PT",
      "column_label": "MBPP",
      "raw_value": "29.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "humaneval",
    "score": 0.445,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '7B-PT', column 'HumanEval' = 44.5%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "7B-PT",
      "column_label": "HumanEval",
      "raw_value": "44.5",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b",
    "bench": "mbpp",
    "score": 0.562,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '7B-PT', column 'MBPP' = 56.2%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "7B-PT",
      "column_label": "MBPP",
      "raw_value": "56.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humaneval",
    "score": 0.561,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "instruct",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '7B-IT', column 'HumanEval' = 56.1%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "7B-IT",
      "column_label": "HumanEval",
      "raw_value": "56.1",
      "scale_divisor": 100
    }
  },
  {
    "model": "codegemma-7b-it",
    "bench": "mbpp",
    "score": 0.542,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "instruct",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row '7B-IT', column 'MBPP' = 54.2%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "7B-IT",
      "column_label": "MBPP",
      "raw_value": "54.2",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "humaneval",
    "score": 0.323,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row 'Gemma 7B PT', column 'HumanEval' = 32.3%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma 7B PT",
      "column_label": "HumanEval",
      "raw_value": "32.3",
      "scale_divisor": 100
    }
  },
  {
    "model": "gemma-7b-base",
    "bench": "mbpp",
    "score": 0.444,
    "source": "https://arxiv.org/html/2406.11409v2",
    "source_id": "paper-2406.11409v2",
    "by": "lab",
    "date": "2024-06-19",
    "mode": "base",
    "note": "CodeGemma paper (arXiv:2406.11409v2, Google) Table 3 'Python coding capability of CodeGemma on de-facto coding benchmarks', row 'Gemma 7B PT', column 'MBPP' = 44.4%. Google's own CodeGemma/Gemma checkpoints, by: lab. Table 5 (math) is collected from other papers and not used.",
    "evidence": {
      "kind": "table",
      "table": "Table 3",
      "row_label": "Gemma 7B PT",
      "column_label": "MBPP",
      "raw_value": "44.4",
      "scale_divisor": 100
    }
  },
  {
    "model": "qwen2-72b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7906,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-06-08",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2-72B-Instruct [bfloat16]' (source=hub, precision=bfloat16); published average 79.06 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 1af63c698f.",
    "raw_value": 79.06,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 82.58,
      "bluex": 75.38,
      "oab_exams": 65.65,
      "assin2_rte": 94.93,
      "assin2_sts": 74.72,
      "faquad_nli": 84.4,
      "hatebr_offensive": 83.94,
      "portuguese_hate_speech": 74.47,
      "tweetsentbr": 75.51
    }
  },
  {
    "model": "llama-3-70b",
    "bench": "open-pt-llm-avg9",
    "score": 0.785,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3-70B-Instruct' (source=hub, precision=bfloat16); published average 78.5 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 359ec69a0f.",
    "raw_value": 78.5,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 80.34,
      "bluex": 73.44,
      "oab_exams": 64.69,
      "assin2_rte": 93.46,
      "assin2_sts": 80.6,
      "faquad_nli": 83.9,
      "hatebr_offensive": 88.94,
      "portuguese_hate_speech": 72.32,
      "tweetsentbr": 68.83
    }
  },
  {
    "model": "mistral-small-3",
    "bench": "open-pt-llm-avg9",
    "score": 0.7723,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-Small-24B-Instruct-2501' (source=hub, precision=bfloat16); published average 77.23 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 010d42b0ae.",
    "raw_value": 77.23,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 78.87,
      "bluex": 73.44,
      "oab_exams": 62.51,
      "assin2_rte": 93.75,
      "assin2_sts": 76.11,
      "faquad_nli": 79.62,
      "hatebr_offensive": 87.36,
      "portuguese_hate_speech": 74.47,
      "tweetsentbr": 68.94
    }
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.7656,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mixtral-8x22B-Instruct-v0.1' (source=hub, precision=bfloat16); published average 76.56 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 95d0639513.",
    "raw_value": 76.56,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 75.44,
      "bluex": 66.62,
      "oab_exams": 57.72,
      "assin2_rte": 94.08,
      "assin2_sts": 84.17,
      "faquad_nli": 80.73,
      "hatebr_offensive": 84.96,
      "portuguese_hate_speech": 71.8,
      "tweetsentbr": 73.56
    }
  },
  {
    "model": "gemma-2-27b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7635,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-02",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-2-27b-it' (source=hub, precision=bfloat16); published average 76.35 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 2d74922e8a.",
    "raw_value": 76.35,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 78.87,
      "bluex": 70.51,
      "oab_exams": 56.36,
      "assin2_rte": 92.38,
      "assin2_sts": 76.34,
      "faquad_nli": 78.72,
      "hatebr_offensive": 88.3,
      "portuguese_hate_speech": 72.13,
      "tweetsentbr": 73.55
    }
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7502,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-06-05",
    "note": "Open PT-LLM Leaderboard row 'THUDM/glm-4-9b-chat [bfloat16]' (source=hub, precision=bfloat16); published average 75.02 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 6f060dcb54; its weight files are identical to the revision pinned in the model entry (bd8234fe5e; safetensors LFS hashes match). Other rows for the same repo not imported: 'THUDM/glm-4-9b-chat [float16]' 75.09.",
    "raw_value": 75.02,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 74.04,
      "bluex": 64.81,
      "oab_exams": 54.4,
      "assin2_rte": 93.14,
      "assin2_sts": 81.91,
      "faquad_nli": 77.82,
      "hatebr_offensive": 82.78,
      "portuguese_hate_speech": 73.14,
      "tweetsentbr": 73.16
    }
  },
  {
    "model": "olmo-2-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7478,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'allenai/OLMo-2-0325-32B-Instruct' (source=hub, precision=bfloat16); published average 74.78 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision b96024342a.",
    "raw_value": 74.78,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 74.88,
      "bluex": 65.37,
      "oab_exams": 50.75,
      "assin2_rte": 93.14,
      "assin2_sts": 72.87,
      "faquad_nli": 80.67,
      "hatebr_offensive": 89.97,
      "portuguese_hate_speech": 74.68,
      "tweetsentbr": 70.68
    }
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7475,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-32B-Chat' (source=hub, precision=bfloat16); published average 74.75 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 0b1785d88b.",
    "raw_value": 74.75,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 75.09,
      "bluex": 65.65,
      "oab_exams": 55.99,
      "assin2_rte": 93.91,
      "assin2_sts": 75.67,
      "faquad_nli": 76.68,
      "hatebr_offensive": 80.98,
      "portuguese_hate_speech": 73.54,
      "tweetsentbr": 75.2
    }
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7474,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-11-12",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-Coder-32B-Instruct' (source=hub, precision=bfloat16); published average 74.74 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision b47205940b.",
    "raw_value": 74.74,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 76.7,
      "bluex": 67.32,
      "oab_exams": 54.49,
      "assin2_rte": 93.22,
      "assin2_sts": 78.33,
      "faquad_nli": 80.86,
      "hatebr_offensive": 78.49,
      "portuguese_hate_speech": 72.78,
      "tweetsentbr": 70.43
    }
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7468,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-110B-Chat [bfloat16]' (source=hub, precision=bfloat16); published average 74.68 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 85f86cec25.",
    "raw_value": 74.68,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 77.96,
      "bluex": 71.49,
      "oab_exams": 57.08,
      "assin2_rte": 92.38,
      "assin2_sts": 76.17,
      "faquad_nli": 80.73,
      "hatebr_offensive": 83.98,
      "portuguese_hate_speech": 62.62,
      "tweetsentbr": 69.68
    }
  },
  {
    "model": "falcon3-10b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7458,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon3-10B-Instruct' (source=hub, precision=bfloat16); published average 74.58 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 8799bc6aec.",
    "raw_value": 74.58,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 72.57,
      "bluex": 62.87,
      "oab_exams": 54.26,
      "assin2_rte": 93.71,
      "assin2_sts": 80.09,
      "faquad_nli": 81.69,
      "hatebr_offensive": 79.37,
      "portuguese_hate_speech": 72.95,
      "tweetsentbr": 73.73
    }
  },
  {
    "model": "command-r-08-2024",
    "bench": "open-pt-llm-avg9",
    "score": 0.7442,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'CohereLabs/c4ai-command-r-08-2024' (source=hub, precision=bfloat16); published average 74.42 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 96b61ca90b.",
    "raw_value": 74.42,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 76.49,
      "bluex": 65.37,
      "oab_exams": 59.59,
      "assin2_rte": 93.3,
      "assin2_sts": 78.41,
      "faquad_nli": 77.27,
      "hatebr_offensive": 81.32,
      "portuguese_hate_speech": 74.28,
      "tweetsentbr": 63.79
    }
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7432,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-18",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-72B-Chat' (source=hub, precision=bfloat16); published average 74.32 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 96a2df029f.",
    "raw_value": 74.32,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 77.05,
      "bluex": 67.59,
      "oab_exams": 55.31,
      "assin2_rte": 92.8,
      "assin2_sts": 78.2,
      "faquad_nli": 79.48,
      "hatebr_offensive": 86.84,
      "portuguese_hate_speech": 62.59,
      "tweetsentbr": 68.98
    }
  },
  {
    "model": "gemma-2-9b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7431,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-02",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-2-9b-it' (source=hub, precision=bfloat16); published average 74.31 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 32e4f3ede2.",
    "raw_value": 74.31,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 74.6,
      "bluex": 63.56,
      "oab_exams": 53.39,
      "assin2_rte": 93.91,
      "assin2_sts": 78.66,
      "faquad_nli": 78.23,
      "hatebr_offensive": 88.75,
      "portuguese_hate_speech": 69.49,
      "tweetsentbr": 68.17
    }
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7413,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-03-05",
    "note": "Open PT-LLM Leaderboard row 'allenai/tulu-2-dpo-70b' (source=hub, precision=bfloat16); published average 74.13 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 0ab5c875f0.",
    "raw_value": 74.13,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 72.36,
      "bluex": 63,
      "oab_exams": 50.62,
      "assin2_rte": 93.17,
      "assin2_sts": 78.34,
      "faquad_nli": 83.4,
      "hatebr_offensive": 82.37,
      "portuguese_hate_speech": 71.93,
      "tweetsentbr": 71.97
    }
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "open-pt-llm-avg9",
    "score": 0.7376,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-Nemo-Instruct-2407' (source=hub, precision=bfloat16); published average 73.76 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision dac9c9e98f; its weight files are identical to the revision pinned in the model entry (04d8a90549; safetensors LFS hashes match).",
    "raw_value": 73.76,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 72.08,
      "bluex": 61.47,
      "oab_exams": 53.8,
      "assin2_rte": 93.22,
      "assin2_sts": 75.18,
      "faquad_nli": 82.34,
      "hatebr_offensive": 77.96,
      "portuguese_hate_speech": 74.3,
      "tweetsentbr": 73.48
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.7314,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-18",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mixtral-8x7B-Instruct-v0.1' (source=hub, precision=bfloat16); published average 73.14 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 125c431e2f.",
    "raw_value": 73.14,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 71.31,
      "bluex": 59.81,
      "oab_exams": 49.57,
      "assin2_rte": 92.64,
      "assin2_sts": 82.74,
      "faquad_nli": 80.09,
      "hatebr_offensive": 78.49,
      "portuguese_hate_speech": 73.13,
      "tweetsentbr": 70.46
    }
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7313,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-11-12",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-Coder-14B-Instruct' (source=hub, precision=bfloat16); published average 73.13 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 1a62978099; its weight files are identical to the revision pinned in the model entry (aedcc2d42b; safetensors LFS hashes match).",
    "raw_value": 73.13,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 73.06,
      "bluex": 66.48,
      "oab_exams": 52.62,
      "assin2_rte": 94.07,
      "assin2_sts": 83.83,
      "faquad_nli": 76.71,
      "hatebr_offensive": 73,
      "portuguese_hate_speech": 72.81,
      "tweetsentbr": 65.6
    }
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.731,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-15",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-34B-Chat' (source=hub, precision=bfloat16); published average 73.1 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision aecc6b2b17; its weight files are identical to the revision pinned in the model entry (fa4ffba162; safetensors LFS hashes match).",
    "raw_value": 73.1,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 69.07,
      "bluex": 66.48,
      "oab_exams": 52.48,
      "assin2_rte": 91.71,
      "assin2_sts": 76.62,
      "faquad_nli": 77.44,
      "hatebr_offensive": 82.11,
      "portuguese_hate_speech": 71.05,
      "tweetsentbr": 70.97
    }
  },
  {
    "model": "aya-expanse-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7303,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-10-27",
    "note": "Open PT-LLM Leaderboard row 'CohereForAI/aya-expanse-32b' (source=hub, precision=float16); published average 73.03 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision c1df2547e1.",
    "raw_value": 73.03,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 73.76,
      "bluex": 65.37,
      "oab_exams": 59.91,
      "assin2_rte": 92.34,
      "assin2_sts": 73.73,
      "faquad_nli": 72.07,
      "hatebr_offensive": 82.52,
      "portuguese_hate_speech": 71.06,
      "tweetsentbr": 66.49
    }
  },
  {
    "model": "falcon3-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7276,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon3-7B-Instruct' (source=hub, precision=bfloat16); published average 72.76 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 5563a370c1; its weight files are identical to the revision pinned in the model entry (1e57a0ecd1; safetensors LFS hashes match).",
    "raw_value": 72.76,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 70.05,
      "bluex": 61.34,
      "oab_exams": 49.16,
      "assin2_rte": 93.09,
      "assin2_sts": 77.2,
      "faquad_nli": 74.63,
      "hatebr_offensive": 85.72,
      "portuguese_hate_speech": 72.74,
      "tweetsentbr": 70.94
    }
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7259,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-23",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-14B-Chat' (source=hub, precision=bfloat16); published average 72.59 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 519cfcc93d.",
    "raw_value": 72.59,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 69.84,
      "bluex": 60.78,
      "oab_exams": 48.43,
      "assin2_rte": 93.22,
      "assin2_sts": 78.09,
      "faquad_nli": 78.86,
      "hatebr_offensive": 83.29,
      "portuguese_hate_speech": 71.29,
      "tweetsentbr": 69.5
    }
  },
  {
    "model": "granite-3-1-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7231,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-08-29",
    "note": "Open PT-LLM Leaderboard row 'ibm-granite/granite-3.1-8b-instruct' (source=hub, precision=float16); published average 72.31 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 4009206d5f.",
    "raw_value": 72.31,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 66.83,
      "bluex": 57.3,
      "oab_exams": 49.61,
      "assin2_rte": 92.63,
      "assin2_sts": 79.93,
      "faquad_nli": 81.51,
      "hatebr_offensive": 74.79,
      "portuguese_hate_speech": 74.82,
      "tweetsentbr": 73.35
    }
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7226,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-21",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3-medium-4k-instruct' (source=hub, precision=bfloat16); published average 72.26 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision bbf1ecfd43.",
    "raw_value": 72.26,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 73.69,
      "bluex": 62.45,
      "oab_exams": 51.98,
      "assin2_rte": 93.54,
      "assin2_sts": 70.31,
      "faquad_nli": 75.02,
      "hatebr_offensive": 83.08,
      "portuguese_hate_speech": 73.98,
      "tweetsentbr": 66.3
    }
  },
  {
    "model": "qwen2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7212,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-06-06",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2-7B-Instruct' (source=hub, precision=bfloat16); published average 72.12 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 41c66b0be1; its weight files are identical to the revision pinned in the model entry (f2826a00ce; safetensors LFS hashes match).",
    "raw_value": 72.12,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 70.82,
      "bluex": 61.06,
      "oab_exams": 49.66,
      "assin2_rte": 93.91,
      "assin2_sts": 75.74,
      "faquad_nli": 75.9,
      "hatebr_offensive": 78.63,
      "portuguese_hate_speech": 70.82,
      "tweetsentbr": 72.54
    }
  },
  {
    "model": "cogito-v1-preview-llama-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7196,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-09",
    "note": "Open PT-LLM Leaderboard row 'deepcogito/cogito-v1-preview-llama-8B' (source=hub, precision=bfloat16); published average 71.96 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 64c42369b3.",
    "raw_value": 71.96,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 71.45,
      "bluex": 58.97,
      "oab_exams": 50.3,
      "assin2_rte": 93.14,
      "assin2_sts": 72.52,
      "faquad_nli": 76.42,
      "hatebr_offensive": 88.69,
      "portuguese_hate_speech": 66.77,
      "tweetsentbr": 69.41
    }
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7174,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-21",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3-medium-128k-instruct' (source=hub, precision=bfloat16); published average 71.74 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision cfe82c6d7d.",
    "raw_value": 71.74,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 71.73,
      "bluex": 61.47,
      "oab_exams": 51.44,
      "assin2_rte": 93.83,
      "assin2_sts": 73.19,
      "faquad_nli": 77.09,
      "hatebr_offensive": 79.87,
      "portuguese_hate_speech": 70.71,
      "tweetsentbr": 66.3
    }
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7167,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-10-13",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-Coder-7B-Instruct' (source=hub, precision=bfloat16); published average 71.67 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 7b148ce7a5; its weight files are identical to the revision pinned in the model entry (c03e6d3582; safetensors LFS hashes match).",
    "raw_value": 71.67,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 67.32,
      "bluex": 56.33,
      "oab_exams": 48.2,
      "assin2_rte": 89.81,
      "assin2_sts": 81.68,
      "faquad_nli": 80.48,
      "hatebr_offensive": 87.51,
      "portuguese_hate_speech": 64.53,
      "tweetsentbr": 69.13
    }
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7144,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-19",
    "note": "Open PT-LLM Leaderboard row 'upstage/SOLAR-10.7B-Instruct-v1.0' (source=hub, precision=float16); published average 71.44 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision cd8ae35fb3.",
    "raw_value": 71.44,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 68.23,
      "bluex": 58.28,
      "oab_exams": 46.06,
      "assin2_rte": 92.77,
      "assin2_sts": 81.67,
      "faquad_nli": 75.43,
      "hatebr_offensive": 80.39,
      "portuguese_hate_speech": 69.13,
      "tweetsentbr": 70.97
    }
  },
  {
    "model": "aya-23-35b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7142,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Open PT-LLM Leaderboard row 'CohereForAI/aya-23-35B' (source=hub, precision=float16); published average 71.42 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 6bb42c6239; its weight files are identical to the revision pinned in the model entry (ce62acc9f0; safetensors LFS hashes match).",
    "raw_value": 71.42,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 70.4,
      "bluex": 60.22,
      "oab_exams": 54.67,
      "assin2_rte": 93.05,
      "assin2_sts": 78.46,
      "faquad_nli": 72.34,
      "hatebr_offensive": 88.6,
      "portuguese_hate_speech": 66.68,
      "tweetsentbr": 58.33
    }
  },
  {
    "model": "ministral-8b-2410",
    "bench": "open-pt-llm-avg9",
    "score": 0.7132,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Ministral-8B-Instruct-2410' (source=hub, precision=bfloat16); published average 71.32 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 199e57c1d6.",
    "raw_value": 71.32,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 68.93,
      "bluex": 56.19,
      "oab_exams": 44.78,
      "assin2_rte": 92.36,
      "assin2_sts": 74.61,
      "faquad_nli": 76.91,
      "hatebr_offensive": 88.02,
      "portuguese_hate_speech": 70.44,
      "tweetsentbr": 69.6
    }
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.7099,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-02",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Codestral-22B-v0.1' (source=hub, precision=bfloat16); published average 70.99 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 352ae5a5cc; its weight files are identical to the revision pinned in the model entry (28b1c1a51d; safetensors LFS hashes match).",
    "raw_value": 70.99,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 64.03,
      "bluex": 54.8,
      "oab_exams": 45.83,
      "assin2_rte": 91.26,
      "assin2_sts": 79.32,
      "faquad_nli": 83.28,
      "hatebr_offensive": 82.54,
      "portuguese_hate_speech": 65.73,
      "tweetsentbr": 72.12
    }
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "open-pt-llm-avg9",
    "score": 0.709,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-03-29",
    "note": "Open PT-LLM Leaderboard row 'Nexusflow/Starling-LM-7B-beta' (source=hub, precision=bfloat16); published average 70.9 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision aa21e7f117.",
    "raw_value": 70.9,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 64.66,
      "bluex": 53.82,
      "oab_exams": 45.42,
      "assin2_rte": 92.57,
      "assin2_sts": 82.47,
      "faquad_nli": 77.49,
      "hatebr_offensive": 83.11,
      "portuguese_hate_speech": 71.37,
      "tweetsentbr": 67.16
    }
  },
  {
    "model": "qwen2-5-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7076,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-3B-Instruct' (source=hub, precision=bfloat16); published average 70.76 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision aa8e725379.",
    "raw_value": 70.76,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 68.65,
      "bluex": 56.88,
      "oab_exams": 47.2,
      "assin2_rte": 92.2,
      "assin2_sts": 79.15,
      "faquad_nli": 77.1,
      "hatebr_offensive": 76.68,
      "portuguese_hate_speech": 68.6,
      "tweetsentbr": 70.42
    }
  },
  {
    "model": "yi-34b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.7076,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-34B-Chat' (source=hub, precision=bfloat16); published average 70.76 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 099f03d148.",
    "raw_value": 70.76,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 71.24,
      "bluex": 63.28,
      "oab_exams": 52.03,
      "assin2_rte": 92.4,
      "assin2_sts": 74.19,
      "faquad_nli": 71.57,
      "hatebr_offensive": 71.98,
      "portuguese_hate_speech": 71.35,
      "tweetsentbr": 68.81
    }
  },
  {
    "model": "phi-3-small-8k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7075,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-21",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3-small-8k-instruct' (source=hub, precision=bfloat16); published average 70.75 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision abd8229ea7.",
    "raw_value": 70.75,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 70.96,
      "bluex": 61.47,
      "oab_exams": 50.25,
      "assin2_rte": 92.09,
      "assin2_sts": 75.88,
      "faquad_nli": 65.01,
      "hatebr_offensive": 85.02,
      "portuguese_hate_speech": 71.41,
      "tweetsentbr": 64.62
    }
  },
  {
    "model": "aya-expanse-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7068,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-10-24",
    "note": "Open PT-LLM Leaderboard row 'CohereForAI/aya-expanse-8b' (source=hub, precision=float16); published average 70.68 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision e46040a1be.",
    "raw_value": 70.68,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 65.92,
      "bluex": 54.38,
      "oab_exams": 47.43,
      "assin2_rte": 91.16,
      "assin2_sts": 77.98,
      "faquad_nli": 74.35,
      "hatebr_offensive": 82.08,
      "portuguese_hate_speech": 70.88,
      "tweetsentbr": 71.91
    }
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "open-pt-llm-avg9",
    "score": 0.7055,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'openchat/openchat-3.5-0106' (source=hub, precision=bfloat16); published average 70.55 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 9619fb7d2a.",
    "raw_value": 70.55,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 64.38,
      "bluex": 52.71,
      "oab_exams": 44.87,
      "assin2_rte": 92.73,
      "assin2_sts": 83.17,
      "faquad_nli": 79.21,
      "hatebr_offensive": 79.31,
      "portuguese_hate_speech": 71.88,
      "tweetsentbr": 66.67
    }
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "open-pt-llm-avg9",
    "score": 0.6959,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-03-04",
    "note": "Open PT-LLM Leaderboard row 'berkeley-nest/Starling-LM-7B-alpha' (source=hub, precision=bfloat16); published average 69.59 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 9dc75f55e3.",
    "raw_value": 69.59,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 64.03,
      "bluex": 55.63,
      "oab_exams": 42.19,
      "assin2_rte": 92.73,
      "assin2_sts": 80.14,
      "faquad_nli": 78.42,
      "hatebr_offensive": 82.22,
      "portuguese_hate_speech": 70.04,
      "tweetsentbr": 60.92
    }
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6922,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-07-05",
    "note": "Open PT-LLM Leaderboard row 'internlm/internlm2_5-7b-chat' (source=hub, precision=bfloat16); published average 69.22 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 6a4dae231b; its weight files are identical to the revision pinned in the model entry (eb72b54168; safetensors LFS hashes match).",
    "raw_value": 69.22,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 65.36,
      "bluex": 55.91,
      "oab_exams": 43.01,
      "assin2_rte": 89.99,
      "assin2_sts": 79.94,
      "faquad_nli": 73.39,
      "hatebr_offensive": 86.49,
      "portuguese_hate_speech": 67.06,
      "tweetsentbr": 61.81
    }
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6865,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-08-30",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Hermes-3-Llama-3.1-8B' (source=hub, precision=bfloat16); published average 68.65 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision aabb745a71; its weight files are identical to the revision pinned in the model entry (896ea440e5; safetensors LFS hashes match).",
    "raw_value": 68.65,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 67.74,
      "bluex": 56.33,
      "oab_exams": 48.47,
      "assin2_rte": 91.32,
      "assin2_sts": 72.53,
      "faquad_nli": 71.36,
      "hatebr_offensive": 85.36,
      "portuguese_hate_speech": 58.79,
      "tweetsentbr": 65.98
    }
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "open-pt-llm-avg9",
    "score": 0.6806,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-22",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-7B-Instruct-v0.3' (source=hub, precision=bfloat16); published average 68.06 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 83e9aa141f.",
    "raw_value": 68.06,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 62.42,
      "bluex": 53.13,
      "oab_exams": 43.6,
      "assin2_rte": 90.27,
      "assin2_sts": 79.43,
      "faquad_nli": 68.67,
      "hatebr_offensive": 83.43,
      "portuguese_hate_speech": 68.25,
      "tweetsentbr": 63.3
    }
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6794,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-15",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-9B-Chat' (source=hub, precision=bfloat16); published average 67.94 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 6afa72fa85; its weight files are identical to the revision pinned in the model entry (1a0fc698cf; safetensors LFS hashes match).",
    "raw_value": 67.94,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 62.42,
      "bluex": 54.8,
      "oab_exams": 44.69,
      "assin2_rte": 88.07,
      "assin2_sts": 75.21,
      "faquad_nli": 69.14,
      "hatebr_offensive": 82.98,
      "portuguese_hate_speech": 66.79,
      "tweetsentbr": 67.33
    }
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6772,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-24",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-7B-Chat' (source=hub, precision=bfloat16); published average 67.72 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 03df580367.",
    "raw_value": 67.72,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 61.58,
      "bluex": 52.29,
      "oab_exams": 44.05,
      "assin2_rte": 89.25,
      "assin2_sts": 70.12,
      "faquad_nli": 72.46,
      "hatebr_offensive": 78.1,
      "portuguese_hate_speech": 73.24,
      "tweetsentbr": 68.41
    }
  },
  {
    "model": "gemma-2-2b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6765,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-08-09",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-2-2b-it [bfloat16]' (source=hub, precision=bfloat16); published average 67.65 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision e48216d900; its weight files are identical to the revision pinned in the model entry (299a8560be; safetensors LFS hashes match).",
    "raw_value": 67.65,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 61.93,
      "bluex": 48.12,
      "oab_exams": 41.82,
      "assin2_rte": 88.78,
      "assin2_sts": 72.2,
      "faquad_nli": 72.75,
      "hatebr_offensive": 88.18,
      "portuguese_hate_speech": 66.63,
      "tweetsentbr": 68.46
    }
  },
  {
    "model": "nous-hermes-2-mixtral-8x7b-dpo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6722,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO [bfloat16]' (source=hub, precision=bfloat16); published average 67.22 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 29972fcc16. Other rows for the same repo not imported: 'NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO [float16]' 67.17.",
    "raw_value": 67.22,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 66.41,
      "bluex": 55.35,
      "oab_exams": 47.29,
      "assin2_rte": 90.24,
      "assin2_sts": 73.49,
      "faquad_nli": 74.98,
      "hatebr_offensive": 76.66,
      "portuguese_hate_speech": 58.61,
      "tweetsentbr": 61.91
    }
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.6681,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3.5-mini-instruct' (source=hub, precision=bfloat16); published average 66.81 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision af0dfb8029; its weight files are identical to the revision pinned in the model entry (2fe1924501; safetensors LFS hashes match).",
    "raw_value": 66.81,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 66.9,
      "bluex": 56.61,
      "oab_exams": 46.92,
      "assin2_rte": 91.95,
      "assin2_sts": 74.32,
      "faquad_nli": 48.75,
      "hatebr_offensive": 84.64,
      "portuguese_hate_speech": 70.85,
      "tweetsentbr": 60.35
    }
  },
  {
    "model": "cogito-v1-preview-llama-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6678,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-04-09",
    "note": "Open PT-LLM Leaderboard row 'deepcogito/cogito-v1-preview-llama-3B' (source=hub, precision=bfloat16); published average 66.78 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 5a0ad6f395.",
    "raw_value": 66.78,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 60.6,
      "bluex": 49.65,
      "oab_exams": 42.64,
      "assin2_rte": 90.39,
      "assin2_sts": 70.26,
      "faquad_nli": 69.18,
      "hatebr_offensive": 84.39,
      "portuguese_hate_speech": 67.51,
      "tweetsentbr": 66.41
    }
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "open-pt-llm-avg9",
    "score": 0.6668,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-18",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-7B-Instruct-v0.2' (source=hub, precision=bfloat16); published average 66.68 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision b70aa86578.",
    "raw_value": 66.68,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 58.92,
      "bluex": 52.16,
      "oab_exams": 39.23,
      "assin2_rte": 90.52,
      "assin2_sts": 74.32,
      "faquad_nli": 66.1,
      "hatebr_offensive": 81.21,
      "portuguese_hate_speech": 70.26,
      "tweetsentbr": 67.42
    }
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6647,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'teknium/OpenHermes-2.5-Mistral-7B' (source=hub, precision=bfloat16); published average 66.47 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 24c0bea14d.",
    "raw_value": 66.47,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 63.89,
      "bluex": 52.85,
      "oab_exams": 43.64,
      "assin2_rte": 90.1,
      "assin2_sts": 68.87,
      "faquad_nli": 71.2,
      "hatebr_offensive": 80.18,
      "portuguese_hate_speech": 69.02,
      "tweetsentbr": 58.46
    }
  },
  {
    "model": "phi-3-mini-4k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.6641,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-23",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3-mini-4k-instruct' (source=hub, precision=bfloat16); published average 66.41 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 3a811845d8. Row dated before Microsoft's 2024-07-01 in-place 'Update model' commit to this repo, so it is the April 2024 checkpoint (the evaluated revision is not in the repo's current commit list).",
    "raw_value": 66.41,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 65.22,
      "bluex": 53.96,
      "oab_exams": 45.69,
      "assin2_rte": 90.64,
      "assin2_sts": 73.6,
      "faquad_nli": 56.06,
      "hatebr_offensive": 84.34,
      "portuguese_hate_speech": 70.82,
      "tweetsentbr": 57.4
    }
  },
  {
    "model": "aya-23-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6625,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Open PT-LLM Leaderboard row 'CohereForAI/aya-23-8B' (source=hub, precision=float16); published average 66.25 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 69ef54bd97; its weight files are identical to the revision pinned in the model entry (89da1a0ed0; safetensors LFS hashes match).",
    "raw_value": 66.25,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 60.46,
      "bluex": 48.4,
      "oab_exams": 43.28,
      "assin2_rte": 91.9,
      "assin2_sts": 78.07,
      "faquad_nli": 65.42,
      "hatebr_offensive": 74.71,
      "portuguese_hate_speech": 64.9,
      "tweetsentbr": 69.09
    }
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "open-pt-llm-avg9",
    "score": 0.6619,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-14",
    "note": "Open PT-LLM Leaderboard row 'HuggingFaceH4/zephyr-7b-alpha' (source=hub, precision=bfloat16); published average 66.19 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 2ce2d02586.",
    "raw_value": 66.19,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 56.26,
      "bluex": 51.04,
      "oab_exams": 40.27,
      "assin2_rte": 90.11,
      "assin2_sts": 72.33,
      "faquad_nli": 69.63,
      "hatebr_offensive": 85.26,
      "portuguese_hate_speech": 65.29,
      "tweetsentbr": 65.49
    }
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.6573,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-23",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-3-mini-128k-instruct' (source=hub, precision=bfloat16); published average 65.73 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 38143357bf. Row dated before Microsoft's 2024-07-01 in-place 'Update model' commit to this repo, so it is the April 2024 checkpoint (the evaluated revision is not in the repo's current commit list).",
    "raw_value": 65.73,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 64.24,
      "bluex": 54.1,
      "oab_exams": 45.65,
      "assin2_rte": 90.4,
      "assin2_sts": 72.13,
      "faquad_nli": 55.67,
      "hatebr_offensive": 83.81,
      "portuguese_hate_speech": 69.58,
      "tweetsentbr": 55.95
    }
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6569,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-18",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-70b-chat-hf' (source=hub, precision=float16); published average 65.69 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision e4b2872127.",
    "raw_value": 65.69,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 62.56,
      "bluex": 47.57,
      "oab_exams": 37.68,
      "assin2_rte": 85.14,
      "assin2_sts": 71.81,
      "faquad_nli": 78.44,
      "hatebr_offensive": 86.92,
      "portuguese_hate_speech": 67.14,
      "tweetsentbr": 53.94
    }
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6557,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-1.5B-Instruct' (source=hub, precision=bfloat16); published average 65.57 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 989aa7980e.",
    "raw_value": 65.57,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 61.58,
      "bluex": 52.57,
      "oab_exams": 44.46,
      "assin2_rte": 87.77,
      "assin2_sts": 75.63,
      "faquad_nli": 72.58,
      "hatebr_offensive": 71.5,
      "portuguese_hate_speech": 67.08,
      "tweetsentbr": 56.98
    }
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "open-pt-llm-avg9",
    "score": 0.6447,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'HuggingFaceH4/zephyr-7b-beta' (source=hub, precision=bfloat16); published average 64.47 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision dc24cabd13.",
    "raw_value": 64.47,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 57.87,
      "bluex": 47.98,
      "oab_exams": 39.32,
      "assin2_rte": 88.36,
      "assin2_sts": 66.78,
      "faquad_nli": 70.18,
      "hatebr_offensive": 81.77,
      "portuguese_hate_speech": 66.59,
      "tweetsentbr": 61.42
    }
  },
  {
    "model": "falcon3-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6413,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon3-3B-Instruct' (source=hub, precision=bfloat16); published average 64.13 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 411bb94318.",
    "raw_value": 64.13,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 55.42,
      "bluex": 46.45,
      "oab_exams": 38.45,
      "assin2_rte": 87.41,
      "assin2_sts": 69.18,
      "faquad_nli": 70.01,
      "hatebr_offensive": 77.58,
      "portuguese_hate_speech": 70.24,
      "tweetsentbr": 62.42
    }
  },
  {
    "model": "olmo-2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6374,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'allenai/OLMo-2-1124-7B-Instruct' (source=hub, precision=bfloat16); published average 63.74 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 470b1fba1a.",
    "raw_value": 63.74,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 55.49,
      "bluex": 48.26,
      "oab_exams": 39.27,
      "assin2_rte": 87.08,
      "assin2_sts": 72.49,
      "faquad_nli": 70.56,
      "hatebr_offensive": 77.21,
      "portuguese_hate_speech": 70.27,
      "tweetsentbr": 53.03
    }
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6334,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-17",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-V2-Lite-Chat' (source=hub, precision=bfloat16); published average 63.34 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision be9443d5ee.",
    "raw_value": 63.34,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 58.08,
      "bluex": 47.71,
      "oab_exams": 40,
      "assin2_rte": 87.41,
      "assin2_sts": 67.77,
      "faquad_nli": 71.28,
      "hatebr_offensive": 70.92,
      "portuguese_hate_speech": 65.81,
      "tweetsentbr": 61.07
    }
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6265,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-6B-Chat' (source=hub, precision=bfloat16); published average 62.65 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision f45214b80a; its weight files are identical to the revision pinned in the model entry (771924d1c8; safetensors LFS hashes match).",
    "raw_value": 62.65,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 50.66,
      "bluex": 46.31,
      "oab_exams": 39.09,
      "assin2_rte": 84.78,
      "assin2_sts": 67.98,
      "faquad_nli": 65.48,
      "hatebr_offensive": 78.81,
      "portuguese_hate_speech": 64.87,
      "tweetsentbr": 65.87
    }
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6188,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-09-21",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-Coder-9B-Chat' (source=hub, precision=bfloat16); published average 61.88 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 356a1f8d4e.",
    "raw_value": 61.88,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 45.42,
      "bluex": 42.56,
      "oab_exams": 35.08,
      "assin2_rte": 79.94,
      "assin2_sts": 76.41,
      "faquad_nli": 65.41,
      "hatebr_offensive": 84.64,
      "portuguese_hate_speech": 61.01,
      "tweetsentbr": 66.41
    }
  },
  {
    "model": "qwen1-5-4b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6134,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-23",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-4B-Chat' (source=hub, precision=bfloat16); published average 61.34 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision f055cb5b26.",
    "raw_value": 61.34,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 50.52,
      "bluex": 46.59,
      "oab_exams": 37.45,
      "assin2_rte": 84.11,
      "assin2_sts": 72.32,
      "faquad_nli": 57.06,
      "hatebr_offensive": 81.93,
      "portuguese_hate_speech": 58.86,
      "tweetsentbr": 63.19
    }
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.5965,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-05-27",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-7B-Instruct-v0.1' (source=hub, precision=bfloat16); published average 59.65 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 73068f3702.",
    "raw_value": 59.65,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 51.99,
      "bluex": 41.59,
      "oab_exams": 34.53,
      "assin2_rte": 86.8,
      "assin2_sts": 65.81,
      "faquad_nli": 60.41,
      "hatebr_offensive": 76.52,
      "portuguese_hate_speech": 63.77,
      "tweetsentbr": 55.39
    }
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.5802,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-13b-chat-hf' (source=hub, precision=bfloat16); published average 58.02 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision a2cb7a712b.",
    "raw_value": 58.02,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 42.76,
      "bluex": 45.62,
      "oab_exams": 39.09,
      "assin2_rte": 77.41,
      "assin2_sts": 58.44,
      "faquad_nli": 68.14,
      "hatebr_offensive": 84.13,
      "portuguese_hate_speech": 56.27,
      "tweetsentbr": 50.34
    }
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.5694,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-08-09",
    "note": "Open PT-LLM Leaderboard row 'internlm/internlm2_5-20b-chat' (source=hub, precision=bfloat16); published average 56.94 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision ef17bde929.",
    "raw_value": 56.94,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 67.32,
      "bluex": 61.34,
      "oab_exams": 45.42,
      "assin2_rte": 59.12,
      "assin2_sts": 82.21,
      "faquad_nli": 42.84,
      "hatebr_offensive": 46.82,
      "portuguese_hate_speech": 42.7,
      "tweetsentbr": 64.66
    }
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.539,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-07",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-1.1-7b-it' (source=hub, precision=bfloat16); published average 53.9 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 689f62a7df.",
    "raw_value": 53.9,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 58.01,
      "bluex": 47.71,
      "oab_exams": 41.5,
      "assin2_rte": 86.83,
      "assin2_sts": 38.52,
      "faquad_nli": 63.6,
      "hatebr_offensive": 67.11,
      "portuguese_hate_speech": 64.43,
      "tweetsentbr": 17.39
    }
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.522,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-7b-chat-hf' (source=hub, precision=bfloat16); published average 52.2 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision f5db02db72.",
    "raw_value": 52.2,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 23.51,
      "bluex": 29.07,
      "oab_exams": 30.66,
      "assin2_rte": 76.92,
      "assin2_sts": 52.56,
      "faquad_nli": 54.84,
      "hatebr_offensive": 85.75,
      "portuguese_hate_speech": 62.15,
      "tweetsentbr": 54.32
    }
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5091,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-0.5B-Instruct' (source=hub, precision=bfloat16); published average 50.91 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 7ae557604a.",
    "raw_value": 50.91,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 36.67,
      "bluex": 35.47,
      "oab_exams": 36.54,
      "assin2_rte": 83.62,
      "assin2_sts": 54.76,
      "faquad_nli": 59.88,
      "hatebr_offensive": 50.89,
      "portuguese_hate_speech": 58.47,
      "tweetsentbr": 41.93
    }
  },
  {
    "model": "gemma-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4802,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-7b-it' (source=hub, precision=bfloat16); published average 48.02 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision f6a08e4923. Other rows for the same repo not imported: 'google/gemma-7b-it (rev: float16)' 49.61.",
    "raw_value": 48.02,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 34.08,
      "bluex": 29.76,
      "oab_exams": 26.83,
      "assin2_rte": 80.63,
      "assin2_sts": 52.97,
      "faquad_nli": 56.18,
      "hatebr_offensive": 76.57,
      "portuguese_hate_speech": 53.91,
      "tweetsentbr": 21.28
    }
  },
  {
    "model": "olmo-2-1b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4504,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'allenai/OLMo-2-0425-1B-Instruct' (source=hub, precision=bfloat16); published average 45.04 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 48d788eca8.",
    "raw_value": 45.04,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 25.26,
      "bluex": 24.9,
      "oab_exams": 28.38,
      "assin2_rte": 77.03,
      "assin2_sts": 37.21,
      "faquad_nli": 41.48,
      "hatebr_offensive": 73.79,
      "portuguese_hate_speech": 58.65,
      "tweetsentbr": 38.67
    }
  },
  {
    "model": "qwen2-0-5b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4437,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-06-07",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2-0.5B-Instruct' (source=hub, precision=bfloat16); published average 44.37 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision c291d6fce4; its weight files are identical to the revision pinned in the model entry (c540970f9e; safetensors LFS hashes match).",
    "raw_value": 44.37,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 30.58,
      "bluex": 34.49,
      "oab_exams": 33.58,
      "assin2_rte": 40.11,
      "assin2_sts": 52.03,
      "faquad_nli": 57.35,
      "hatebr_offensive": 72.34,
      "portuguese_hate_speech": 47.77,
      "tweetsentbr": 31.06
    }
  },
  {
    "model": "falcon3-1b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4423,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon3-1B-Instruct' (source=hub, precision=bfloat16); published average 44.23 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 28ba225197.",
    "raw_value": 44.23,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 34.57,
      "bluex": 31.57,
      "oab_exams": 30.84,
      "assin2_rte": 76.08,
      "assin2_sts": 21.2,
      "faquad_nli": 51.95,
      "hatebr_offensive": 69.28,
      "portuguese_hate_speech": 49.44,
      "tweetsentbr": 33.15
    }
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3549,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-04-07",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-1.1-2b-it' (source=hub, precision=bfloat16); published average 35.49 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 28d20bc156.",
    "raw_value": 35.49,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 28.41,
      "bluex": 24.9,
      "oab_exams": 30.75,
      "assin2_rte": 59.22,
      "assin2_sts": 1.29,
      "faquad_nli": 45.1,
      "hatebr_offensive": 54.59,
      "portuguese_hate_speech": 23.71,
      "tweetsentbr": 51.47
    }
  },
  {
    "model": "gemma-2b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3425,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-2b-it' (source=hub, precision=bfloat16); published average 34.25 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 9642e777f2.",
    "raw_value": 34.25,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 28.76,
      "bluex": 25.87,
      "oab_exams": 28.38,
      "assin2_rte": 57.17,
      "assin2_sts": 5.51,
      "faquad_nli": 55.2,
      "hatebr_offensive": 44.55,
      "portuguese_hate_speech": 23.59,
      "tweetsentbr": 39.2
    }
  },
  {
    "model": "tinyllama-1-1b-chat-v1-0",
    "bench": "open-pt-llm-avg9",
    "score": 0.2822,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "source_id": "open-pt-llm-leaderboard",
    "by": "independent",
    "date": "2024-02-22",
    "note": "Open PT-LLM Leaderboard row 'TinyLlama/TinyLlama-1.1B-Chat-v1.0' (source=hub, precision=bfloat16); published average 28.22 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 77e23968ee; its weight files are identical to the revision pinned in the model entry (fe8a4ea1ff; safetensors LFS hashes match).",
    "raw_value": 28.22,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 17.07,
      "bluex": 21.56,
      "oab_exams": 23.23,
      "assin2_rte": 43.9,
      "assin2_sts": 0.52,
      "faquad_nli": 43.97,
      "hatebr_offensive": 33.33,
      "portuguese_hate_speech": 42.33,
      "tweetsentbr": 28.04
    }
  },
  {
    "model": "llama-3-70b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.659171,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Meta-Llama-3-70B-Instruct' (query id meta-llama/Meta-Llama-3-70B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 65.9171 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 50fd307e57.",
    "raw_value": 65.917143,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 79.34,
      "GSM8K": 28.7,
      "DC-Homograph": 67.59,
      "MC-Homograph": 94.01,
      "PiQA": 89.59,
      "Proverb-Quiz": 72.43,
      "VerbEval": 59.41,
      "Winogrande": 75.91,
      "Arc-Challenge": 88.35,
      "Arc-Easy": 95.19,
      "Feqh": 36.57,
      "Hallucination (Truthfulness)": 54.64,
      "P-Hellaswag": 82.51,
      "Law": 44.33,
      "AUT Multiple Choice": 56.8,
      "Parsi Literature": 36.94,
      "BoolQA": 92.2,
      "Reading Comprehension": 66.88,
      "PartExpert": 51.12,
      "MMLU Pro": 38.7,
      "Iranian Social Norms": 73.05
    }
  },
  {
    "model": "qwen2-72b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.651386,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2-72B-Instruct' (query id Qwen/Qwen2-72B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 65.1386 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision c867f763ef.",
    "raw_value": 65.138571,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 75.26,
      "GSM8K": 35.6,
      "DC-Homograph": 72.22,
      "MC-Homograph": 91.71,
      "PiQA": 89.79,
      "Proverb-Quiz": 70,
      "VerbEval": 53.26,
      "Winogrande": 80.87,
      "Arc-Challenge": 88.03,
      "Arc-Easy": 94.65,
      "Feqh": 31.43,
      "Hallucination (Truthfulness)": 62.76,
      "P-Hellaswag": 83.1,
      "Law": 38,
      "AUT Multiple Choice": 55,
      "Parsi Literature": 35.78,
      "BoolQA": 88.8,
      "Reading Comprehension": 63.97,
      "PartExpert": 48.63,
      "MMLU Pro": 34.6,
      "Iranian Social Norms": 74.45
    }
  },
  {
    "model": "gemma-2-27b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.608367,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-2-27b-it' (query id google/gemma-2-27b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 60.8367 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision aaf20e6b9f.",
    "raw_value": 60.836667,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 68.11,
      "GSM8K": 26.7,
      "DC-Homograph": 60.19,
      "MC-Homograph": 91.24,
      "PiQA": 89.69,
      "Proverb-Quiz": 73.51,
      "VerbEval": 61.16,
      "Winogrande": 76.44,
      "Arc-Challenge": 86.75,
      "Arc-Easy": 94.22,
      "Feqh": 24,
      "Hallucination (Truthfulness)": 13.05,
      "P-Hellaswag": 83.69,
      "Law": 34.67,
      "AUT Multiple Choice": 50.8,
      "Parsi Literature": 35.91,
      "BoolQA": 89.8,
      "Reading Comprehension": 56.76,
      "PartExpert": 46.6,
      "MMLU Pro": 36.9,
      "Iranian Social Norms": 77.38
    }
  },
  {
    "model": "gemma-2-9b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.6061,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-2-9b-it' (query id google/gemma-2-9b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 60.6100 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 11c9b309ab.",
    "raw_value": 60.61,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 64.03,
      "GSM8K": 17.4,
      "DC-Homograph": 59.26,
      "MC-Homograph": 90.55,
      "PiQA": 87.09,
      "Proverb-Quiz": 69.19,
      "VerbEval": 58.25,
      "Winogrande": 72.01,
      "Arc-Challenge": 84.29,
      "Arc-Easy": 93.16,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 50.58,
      "P-Hellaswag": 80.82,
      "Law": 33.67,
      "AUT Multiple Choice": 48.5,
      "Parsi Literature": 38.1,
      "BoolQA": 89.7,
      "Reading Comprehension": 56.43,
      "PartExpert": 43.03,
      "MMLU Pro": 33.2,
      "Iranian Social Norms": 73.84
    }
  },
  {
    "model": "command-r-08-2024",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.597895,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-r-08-2024' (query id CohereLabs/c4ai-command-r-08-2024, precision F16); Average Accuracy = mean of the 21 published task scores = 59.7895 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 96b61ca90b.",
    "raw_value": 59.789524,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 72.45,
      "GSM8K": 18.1,
      "DC-Homograph": 50.93,
      "MC-Homograph": 80.88,
      "PiQA": 86.09,
      "Proverb-Quiz": 77.57,
      "VerbEval": 56.26,
      "Winogrande": 67.58,
      "Arc-Challenge": 81.09,
      "Arc-Easy": 89.84,
      "Feqh": 33.14,
      "Hallucination (Truthfulness)": 49.16,
      "P-Hellaswag": 79.65,
      "Law": 36.33,
      "AUT Multiple Choice": 51.4,
      "Parsi Literature": 32.95,
      "BoolQA": 81.4,
      "Reading Comprehension": 66.03,
      "PartExpert": 43.04,
      "MMLU Pro": 30.1,
      "Iranian Social Norms": 71.59
    }
  },
  {
    "model": "aya-23-35b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.552333,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'aya-23-35B' (query id CohereLabs/aya-23-35B, precision F16); Average Accuracy = mean of the 21 published task scores = 55.2333 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 5e72bd5ad8.",
    "raw_value": 55.233333,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 63.27,
      "GSM8K": 10,
      "DC-Homograph": 55.56,
      "MC-Homograph": 83.64,
      "PiQA": 89.49,
      "Proverb-Quiz": 67.03,
      "VerbEval": 47.32,
      "Winogrande": 65.81,
      "Arc-Challenge": 77.56,
      "Arc-Easy": 90.16,
      "Feqh": 30.29,
      "Hallucination (Truthfulness)": 11.72,
      "P-Hellaswag": 79.87,
      "Law": 32,
      "AUT Multiple Choice": 48.7,
      "Parsi Literature": 31.92,
      "BoolQA": 86.2,
      "Reading Comprehension": 62.82,
      "PartExpert": 37.44,
      "MMLU Pro": 24.1,
      "Iranian Social Norms": 65
    }
  },
  {
    "model": "llama-3-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.504081,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Meta-Llama-3-8B-Instruct' (query id meta-llama/Meta-Llama-3-8B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 50.4081 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 8afb486c1d.",
    "raw_value": 50.408095,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 52.04,
      "GSM8K": 10.4,
      "DC-Homograph": 41.67,
      "MC-Homograph": 81.11,
      "PiQA": 70.97,
      "Proverb-Quiz": 42.97,
      "VerbEval": 38.93,
      "Winogrande": 56.95,
      "Arc-Challenge": 66.77,
      "Arc-Easy": 76.47,
      "Feqh": 33.71,
      "Hallucination (Truthfulness)": 33.23,
      "P-Hellaswag": 76.71,
      "Law": 32,
      "AUT Multiple Choice": 45,
      "Parsi Literature": 29.99,
      "BoolQA": 82.5,
      "Reading Comprehension": 54.79,
      "PartExpert": 36.3,
      "MMLU Pro": 26,
      "Iranian Social Norms": 70.06
    }
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.503762,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Hermes-3-Llama-3.1-8B' (query id NousResearch/Hermes-3-Llama-3.1-8B, precision BF16); Average Accuracy = mean of the 21 published task scores = 50.3762 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 896ea440e5.",
    "raw_value": 50.37619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 49.49,
      "GSM8K": 10.2,
      "DC-Homograph": 44.44,
      "MC-Homograph": 79.72,
      "PiQA": 70.37,
      "Proverb-Quiz": 47.84,
      "VerbEval": 48.94,
      "Winogrande": 55.18,
      "Arc-Challenge": 65.28,
      "Arc-Easy": 78.07,
      "Feqh": 30.29,
      "Hallucination (Truthfulness)": 45.2,
      "P-Hellaswag": 73.99,
      "Law": 31.67,
      "AUT Multiple Choice": 42.1,
      "Parsi Literature": 30.63,
      "BoolQA": 83.5,
      "Reading Comprehension": 56.4,
      "PartExpert": 35.61,
      "MMLU Pro": 24.1,
      "Iranian Social Norms": 54.88
    }
  },
  {
    "model": "qwen2-7b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.49959,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2-7B-Instruct' (query id Qwen/Qwen2-7B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 49.9590 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision f2826a00ce.",
    "raw_value": 49.959048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 52.04,
      "GSM8K": 14.5,
      "DC-Homograph": 54.63,
      "MC-Homograph": 72.81,
      "PiQA": 70.97,
      "Proverb-Quiz": 50.54,
      "VerbEval": 40.62,
      "Winogrande": 60.94,
      "Arc-Challenge": 69.12,
      "Arc-Easy": 80.75,
      "Feqh": 28,
      "Hallucination (Truthfulness)": 25.93,
      "P-Hellaswag": 76.71,
      "Law": 28.33,
      "AUT Multiple Choice": 40.4,
      "Parsi Literature": 31.4,
      "BoolQA": 79,
      "Reading Comprehension": 50.14,
      "PartExpert": 36.31,
      "MMLU Pro": 23.8,
      "Iranian Social Norms": 62.2
    }
  },
  {
    "model": "aya-23-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.48509,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'aya-23-8B' (query id CohereLabs/aya-23-8B, precision F16); Average Accuracy = mean of the 21 published task scores = 48.5090 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 2a1a63b24a.",
    "raw_value": 48.509048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 52.3,
      "GSM8K": 6.1,
      "DC-Homograph": 52.78,
      "MC-Homograph": 76.27,
      "PiQA": 80.78,
      "Proverb-Quiz": 44.32,
      "VerbEval": 39.3,
      "Winogrande": 57.13,
      "Arc-Challenge": 63.68,
      "Arc-Easy": 81.39,
      "Feqh": 29.14,
      "Hallucination (Truthfulness)": 0.6,
      "P-Hellaswag": 75.83,
      "Law": 28.33,
      "AUT Multiple Choice": 42.9,
      "Parsi Literature": 31.27,
      "BoolQA": 72.3,
      "Reading Comprehension": 60.31,
      "PartExpert": 33.33,
      "MMLU Pro": 19.9,
      "Iranian Social Norms": 70.73
    }
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.466952,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-Nemo-Instruct-2407' (query id mistralai/Mistral-Nemo-Instruct-2407, precision BF16); Average Accuracy = mean of the 21 published task scores = 46.6952 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 04d8a90549.",
    "raw_value": 46.695238,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 41.07,
      "GSM8K": 12.8,
      "DC-Homograph": 51.85,
      "MC-Homograph": 80.88,
      "PiQA": 71.57,
      "Proverb-Quiz": 40,
      "VerbEval": 37.13,
      "Winogrande": 37.13,
      "Arc-Challenge": 65.92,
      "Arc-Easy": 79.89,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 3.15,
      "P-Hellaswag": 76.93,
      "Law": 30.67,
      "AUT Multiple Choice": 38.7,
      "Parsi Literature": 30.89,
      "BoolQA": 69,
      "Reading Comprehension": 62.32,
      "PartExpert": 33.46,
      "MMLU Pro": 20.4,
      "Iranian Social Norms": 67.13
    }
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.461176,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mixtral-8x7B-Instruct-v0.1' (query id mistralai/Mixtral-8x7B-Instruct-v0.1, precision BF16); Average Accuracy = mean of the 21 published task scores = 46.1176 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision eba92302a2.",
    "raw_value": 46.117619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 35.97,
      "GSM8K": 11.4,
      "DC-Homograph": 50,
      "MC-Homograph": 68.43,
      "PiQA": 66.97,
      "Proverb-Quiz": 43.24,
      "VerbEval": 38.75,
      "Winogrande": 55.89,
      "Arc-Challenge": 55.88,
      "Arc-Easy": 65.45,
      "Feqh": 26.29,
      "Hallucination (Truthfulness)": 36.56,
      "P-Hellaswag": 74.06,
      "Law": 28,
      "AUT Multiple Choice": 33.2,
      "Parsi Literature": 27.16,
      "BoolQA": 76.5,
      "Reading Comprehension": 59.79,
      "PartExpert": 32.46,
      "MMLU Pro": 23.2,
      "Iranian Social Norms": 59.27
    }
  },
  {
    "model": "gemma-2-2b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.443743,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gemma-2-2b-it' (query id google/gemma-2-2b-it, precision BF16); Average Accuracy = mean of the 21 published task scores = 44.3743 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 299a8560be.",
    "raw_value": 44.374286,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 32.91,
      "GSM8K": 6.4,
      "DC-Homograph": 47.22,
      "MC-Homograph": 74.65,
      "PiQA": 66.87,
      "Proverb-Quiz": 45.68,
      "VerbEval": 36.18,
      "Winogrande": 54.74,
      "Arc-Challenge": 57.91,
      "Arc-Easy": 70.48,
      "Feqh": 25.71,
      "Hallucination (Truthfulness)": 39.02,
      "P-Hellaswag": 69.88,
      "Law": 32.67,
      "AUT Multiple Choice": 36.9,
      "Parsi Literature": 30.76,
      "BoolQA": 72.4,
      "Reading Comprehension": 41.79,
      "PartExpert": 31.31,
      "MMLU Pro": 18.2,
      "Iranian Social Norms": 40.18
    }
  },
  {
    "model": "mistral-small-2409",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.440476,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-Small-Instruct-2409' (query id mistralai/Mistral-Small-Instruct-2409, precision BF16); Average Accuracy = mean of the 21 published task scores = 44.0476 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 4600506f6b.",
    "raw_value": 44.047619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 34.95,
      "GSM8K": 12.2,
      "DC-Homograph": 49.07,
      "MC-Homograph": 78.11,
      "PiQA": 61.36,
      "Proverb-Quiz": 37.3,
      "VerbEval": 31.8,
      "Winogrande": 31.8,
      "Arc-Challenge": 55.24,
      "Arc-Easy": 61.6,
      "Feqh": 20.57,
      "Hallucination (Truthfulness)": 28.1,
      "P-Hellaswag": 75.46,
      "Law": 28.67,
      "AUT Multiple Choice": 34.6,
      "Parsi Literature": 28.7,
      "BoolQA": 84.8,
      "Reading Comprehension": 64.56,
      "PartExpert": 31.52,
      "MMLU Pro": 21.3,
      "Iranian Social Norms": 53.29
    }
  },
  {
    "model": "qwen2-5-3b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.434624,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-3B-Instruct' (query id Qwen/Qwen2.5-3B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 43.4624 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision aa8e725379.",
    "raw_value": 43.462381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 40.56,
      "GSM8K": 34.1,
      "DC-Homograph": 54.63,
      "MC-Homograph": 70.97,
      "PiQA": 62.16,
      "Proverb-Quiz": 41.08,
      "VerbEval": 38.66,
      "Winogrande": 38.66,
      "Arc-Challenge": 51.5,
      "Arc-Easy": 67.27,
      "Feqh": 24.57,
      "Hallucination (Truthfulness)": 33.09,
      "P-Hellaswag": 66.5,
      "Law": 30.33,
      "AUT Multiple Choice": 38.9,
      "Parsi Literature": 29.73,
      "BoolQA": 64.9,
      "Reading Comprehension": 40.41,
      "PartExpert": 33.35,
      "MMLU Pro": 21.1,
      "Iranian Social Norms": 30.24
    }
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.409967,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-7B-Instruct-v0.3' (query id mistralai/Mistral-7B-Instruct-v0.3, precision BF16); Average Accuracy = mean of the 21 published task scores = 40.9967 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 0d4b76e1ef.",
    "raw_value": 40.996667,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 30.61,
      "GSM8K": 5.6,
      "DC-Homograph": 47.22,
      "MC-Homograph": 64.52,
      "PiQA": 59.96,
      "Proverb-Quiz": 38.38,
      "VerbEval": 32.57,
      "Winogrande": 50.93,
      "Arc-Challenge": 40.49,
      "Arc-Easy": 48.88,
      "Feqh": 33.71,
      "Hallucination (Truthfulness)": 28.8,
      "P-Hellaswag": 68.99,
      "Law": 24.67,
      "AUT Multiple Choice": 30.5,
      "Parsi Literature": 27.41,
      "BoolQA": 68.7,
      "Reading Comprehension": 55.47,
      "PartExpert": 29.19,
      "MMLU Pro": 17.5,
      "Iranian Social Norms": 56.83
    }
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.393619,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-7B-Instruct-v0.2' (query id mistralai/Mistral-7B-Instruct-v0.2, precision BF16); Average Accuracy = mean of the 21 published task scores = 39.3619 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 63a8b08189.",
    "raw_value": 39.361905,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 32.65,
      "GSM8K": 4.5,
      "DC-Homograph": 45.37,
      "MC-Homograph": 58.29,
      "PiQA": 56.46,
      "Proverb-Quiz": 35.41,
      "VerbEval": 32.05,
      "Winogrande": 50.04,
      "Arc-Challenge": 37.07,
      "Arc-Easy": 45.03,
      "Feqh": 24.57,
      "Hallucination (Truthfulness)": 54.96,
      "P-Hellaswag": 63.56,
      "Law": 27.33,
      "AUT Multiple Choice": 27.1,
      "Parsi Literature": 24.32,
      "BoolQA": 64,
      "Reading Comprehension": 48.19,
      "PartExpert": 28.26,
      "MMLU Pro": 16.4,
      "Iranian Social Norms": 51.04
    }
  },
  {
    "model": "qwen2-1-5b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.367724,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2-1.5B-Instruct' (query id Qwen/Qwen2-1.5B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 36.7724 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision ba1cf1846d.",
    "raw_value": 36.772381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 28.06,
      "GSM8K": 3.8,
      "DC-Homograph": 50.93,
      "MC-Homograph": 56.45,
      "PiQA": 56.16,
      "Proverb-Quiz": 36.49,
      "VerbEval": 32.57,
      "Winogrande": 32.57,
      "Arc-Challenge": 37.18,
      "Arc-Easy": 48.98,
      "Feqh": 28.57,
      "Hallucination (Truthfulness)": 30.69,
      "P-Hellaswag": 64.07,
      "Law": 28.67,
      "AUT Multiple Choice": 33.1,
      "Parsi Literature": 26.77,
      "BoolQA": 63.8,
      "Reading Comprehension": 38.91,
      "PartExpert": 29.96,
      "MMLU Pro": 15.1,
      "Iranian Social Norms": 29.39
    }
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.367633,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-1.5B-Instruct' (query id Qwen/Qwen2.5-1.5B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 36.7633 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 989aa7980e.",
    "raw_value": 36.763333,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 28.32,
      "GSM8K": 5.7,
      "DC-Homograph": 47.22,
      "MC-Homograph": 61.29,
      "PiQA": 56.16,
      "Proverb-Quiz": 35.14,
      "VerbEval": 31.99,
      "Winogrande": 31.99,
      "Arc-Challenge": 42.63,
      "Arc-Easy": 48.02,
      "Feqh": 21.71,
      "Hallucination (Truthfulness)": 15.7,
      "P-Hellaswag": 70.1,
      "Law": 24.67,
      "AUT Multiple Choice": 34.2,
      "Parsi Literature": 27.67,
      "BoolQA": 70,
      "Reading Comprehension": 42.47,
      "PartExpert": 30.07,
      "MMLU Pro": 18.2,
      "Iranian Social Norms": 28.78
    }
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.36489,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-7B-Instruct-v0.1' (query id mistralai/Mistral-7B-Instruct-v0.1, precision BF16); Average Accuracy = mean of the 21 published task scores = 36.4890 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision ec5deb64f2.",
    "raw_value": 36.489048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 27.3,
      "GSM8K": 4.5,
      "DC-Homograph": 53.7,
      "MC-Homograph": 40.78,
      "PiQA": 55.16,
      "Proverb-Quiz": 33.24,
      "VerbEval": 26.6,
      "Winogrande": 51.46,
      "Arc-Challenge": 29.7,
      "Arc-Easy": 33.48,
      "Feqh": 30.86,
      "Hallucination (Truthfulness)": 47.41,
      "P-Hellaswag": 65.61,
      "Law": 23.67,
      "AUT Multiple Choice": 28.6,
      "Parsi Literature": 24.45,
      "BoolQA": 63.6,
      "Reading Comprehension": 34.45,
      "PartExpert": 27.17,
      "MMLU Pro": 13.8,
      "Iranian Social Norms": 50.73
    }
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.333295,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2.5-0.5B-Instruct' (query id Qwen/Qwen2.5-0.5B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 33.3295 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision 7ae557604a.",
    "raw_value": 33.329524,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 28.32,
      "GSM8K": 2.5,
      "DC-Homograph": 53.7,
      "MC-Homograph": 43.09,
      "PiQA": 53.35,
      "Proverb-Quiz": 34.32,
      "VerbEval": 28.41,
      "Winogrande": 28.41,
      "Arc-Challenge": 28.42,
      "Arc-Easy": 35.08,
      "Feqh": 26.86,
      "Hallucination (Truthfulness)": 63.75,
      "P-Hellaswag": 51.73,
      "Law": 24.67,
      "AUT Multiple Choice": 26.9,
      "Parsi Literature": 27.28,
      "BoolQA": 46.4,
      "Reading Comprehension": 25.22,
      "PartExpert": 27.01,
      "MMLU Pro": 14.2,
      "Iranian Social Norms": 30.3
    }
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.326457,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-2-70b-chat-hf' (query id meta-llama/Llama-2-70b-chat-hf, precision F16); Average Accuracy = mean of the 21 published task scores = 32.6457 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision e9149a1280.",
    "raw_value": 32.645714,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 23.98,
      "GSM8K": 4.9,
      "DC-Homograph": 44.44,
      "MC-Homograph": 53.46,
      "PiQA": 54.75,
      "Proverb-Quiz": 28.38,
      "VerbEval": 24.03,
      "Winogrande": 49.16,
      "Arc-Challenge": 31.94,
      "Arc-Easy": 33.26,
      "Feqh": 30.29,
      "Hallucination (Truthfulness)": 2.18,
      "P-Hellaswag": 63.7,
      "Law": 27.67,
      "AUT Multiple Choice": 27.2,
      "Parsi Literature": 23.29,
      "BoolQA": 67.6,
      "Reading Comprehension": 0,
      "PartExpert": 26.58,
      "MMLU Pro": 17.1,
      "Iranian Social Norms": 51.65
    }
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.317343,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-2-7b-chat-hf' (query id meta-llama/Llama-2-7b-chat-hf, precision F16); Average Accuracy = mean of the 21 published task scores = 31.7343 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision f5db02db72.",
    "raw_value": 31.734286,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 26.53,
      "GSM8K": 3.3,
      "DC-Homograph": 46.3,
      "MC-Homograph": 37.56,
      "PiQA": 49.25,
      "Proverb-Quiz": 33.51,
      "VerbEval": 26.11,
      "Winogrande": 48.8,
      "Arc-Challenge": 27.99,
      "Arc-Easy": 29.52,
      "Feqh": 25.71,
      "Hallucination (Truthfulness)": 20.23,
      "P-Hellaswag": 57.75,
      "Law": 26,
      "AUT Multiple Choice": 26.4,
      "Parsi Literature": 25.35,
      "BoolQA": 64.1,
      "Reading Comprehension": 0,
      "PartExpert": 26.2,
      "MMLU Pro": 15.2,
      "Iranian Social Norms": 50.61
    }
  },
  {
    "model": "qwen2-0-5b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.30629,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2-0.5B-Instruct' (query id Qwen/Qwen2-0.5B-Instruct, precision BF16); Average Accuracy = mean of the 21 published task scores = 30.6290 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision c540970f9e.",
    "raw_value": 30.629048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 27.3,
      "GSM8K": 3.5,
      "DC-Homograph": 42.59,
      "MC-Homograph": 41.01,
      "PiQA": 54.75,
      "Proverb-Quiz": 32.43,
      "VerbEval": 27.15,
      "Winogrande": 27.15,
      "Arc-Challenge": 28.63,
      "Arc-Easy": 31.76,
      "Feqh": 25.14,
      "Hallucination (Truthfulness)": 22.78,
      "P-Hellaswag": 50.48,
      "Law": 27.33,
      "AUT Multiple Choice": 28.1,
      "Parsi Literature": 26.38,
      "BoolQA": 37.1,
      "Reading Comprehension": 22.28,
      "PartExpert": 27.35,
      "MMLU Pro": 13.6,
      "Iranian Social Norms": 46.4
    }
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.294033,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "source_id": "open-persian-llm-leaderboard",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-2-13b-chat-hf' (query id meta-llama/Llama-2-13b-chat-hf, precision F32); Average Accuracy = mean of the 21 published task scores = 29.4033 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Evaluated revision a2cb7a712b.",
    "raw_value": 29.403333,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 28.57,
      "GSM8K": 3.3,
      "DC-Homograph": 44.44,
      "MC-Homograph": 37.33,
      "PiQA": 50.15,
      "Proverb-Quiz": 30,
      "VerbEval": 26.17,
      "Winogrande": 49.16,
      "Arc-Challenge": 28.42,
      "Arc-Easy": 29.52,
      "Feqh": 22.86,
      "Hallucination (Truthfulness)": 30.26,
      "P-Hellaswag": 36.96,
      "Law": 19.67,
      "AUT Multiple Choice": 25.4,
      "Parsi Literature": 26.13,
      "BoolQA": 63.9,
      "Reading Comprehension": 0,
      "PartExpert": 25.94,
      "MMLU Pro": 13.8,
      "Iranian Social Norms": 25.49
    }
  },
  {
    "model": "llama-3-70b",
    "bench": "cptub-speakleash-avg",
    "score": 0.756374,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Meta-Llama-3-70B-Instruct'; Space Average = mean(Sentiment 4.135, Language understanding 3.820, Phraseology 3.465, Tricky questions 3.708) = 3.7819 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.78187,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-72b",
    "bench": "cptub-speakleash-avg",
    "score": 0.73063,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2-72B-Instruct'; Space Average = mean(Sentiment 3.763, Language understanding 3.890, Phraseology 3.280, Tricky questions 3.680) = 3.6531 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.653149,
    "scale_divisor": 5
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "cptub-speakleash-avg",
    "score": 0.631645,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen1.5-72B-Chat'; Space Average = mean(Sentiment 3.474, Language understanding 3.515, Phraseology 2.975, Tricky questions 2.669) = 3.1582 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.158225,
    "scale_divisor": 5
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "cptub-speakleash-avg",
    "score": 0.590394,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'THUDM/glm-4-9b-chat'; Space Average = mean(Sentiment 3.590, Language understanding 3.455, Phraseology 2.780, Tricky questions 1.983) = 2.9520 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.951972,
    "scale_divisor": 5
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "cptub-speakleash-avg",
    "score": 0.588046,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Nemo-Instruct-2407'; Space Average = mean(Sentiment 3.641, Language understanding 3.290, Phraseology 2.740, Tricky questions 2.090) = 2.9402 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.940228,
    "scale_divisor": 5
  },
  {
    "model": "llama-3-8b",
    "bench": "cptub-speakleash-avg",
    "score": 0.599793,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'meta-llama/Meta-Llama-3-8B-Instruct'; Space Average = mean(Sentiment 3.333, Language understanding 3.150, Phraseology 3.035, Tricky questions 2.478) = 2.9990 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.998965,
    "scale_divisor": 5
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.576327,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'upstage/SOLAR-10.7B-Instruct-v1.0'; Space Average = mean(Sentiment 2.968, Language understanding 3.180, Phraseology 3.255, Tricky questions 2.124) = 2.8816 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.881636,
    "scale_divisor": 5
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "cptub-speakleash-avg",
    "score": 0.545772,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mixtral-8x7B-Instruct-v0.1'; Space Average = mean(Sentiment 3.058, Language understanding 3.175, Phraseology 2.885, Tricky questions 1.798) = 2.7289 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.728861,
    "scale_divisor": 5
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "cptub-speakleash-avg",
    "score": 0.552784,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-7B-Instruct-v0.3'; Space Average = mean(Sentiment 3.327, Language understanding 3.060, Phraseology 2.680, Tricky questions 1.989) = 2.7639 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.763922,
    "scale_divisor": 5
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "cptub-speakleash-avg",
    "score": 0.525873,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'berkeley-nest/Starling-LM-7B-alpha'; Space Average = mean(Sentiment 3.058, Language understanding 2.925, Phraseology 2.855, Tricky questions 1.680) = 2.6294 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.629367,
    "scale_divisor": 5
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "cptub-speakleash-avg",
    "score": 0.525547,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'openchat/openchat-3.5-0106'; Space Average = mean(Sentiment 3.160, Language understanding 2.835, Phraseology 2.555, Tricky questions 1.961) = 2.6277 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.627733,
    "scale_divisor": 5
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "cptub-speakleash-avg",
    "score": 0.466346,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry '01-ai/Yi-1.5-34B-Chat'; Space Average = mean(Sentiment 3.077, Language understanding 2.870, Phraseology 2.380, Tricky questions 1.000) = 2.3317 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.331731,
    "scale_divisor": 5
  },
  {
    "model": "gemma-2-2b",
    "bench": "cptub-speakleash-avg",
    "score": 0.530296,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'google/gemma-2-2b-it'; Space Average = mean(Sentiment 3.397, Language understanding 2.900, Phraseology 2.095, Tricky questions 2.213) = 2.6515 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.65148,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.500635,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-3B-Instruct'; Space Average = mean(Sentiment 2.949, Language understanding 2.455, Phraseology 2.800, Tricky questions 1.809) = 2.5032 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.503177,
    "scale_divisor": 5
  },
  {
    "model": "hermes-3-llama-3-2-3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.461292,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'NousResearch/Hermes-3-Llama-3.2-3B'; Space Average = mean(Sentiment 2.615, Language understanding 2.705, Phraseology 2.765, Tricky questions 1.140) = 2.3065 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.306459,
    "scale_divisor": 5
  },
  {
    "model": "granite-3-1-2b",
    "bench": "cptub-speakleash-avg",
    "score": 0.389091,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'ibm-granite/granite-3.1-2b-instruct'; Space Average = mean(Sentiment 3.077, Language understanding 2.235, Phraseology 1.880, Tricky questions 0.590) = 1.9455 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.945453,
    "scale_divisor": 5
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.402042,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'microsoft/Phi-3.5-mini-instruct'; Space Average = mean(Sentiment 2.436, Language understanding 2.135, Phraseology 2.425, Tricky questions 1.045) = 2.0102 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 2.01021,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "cptub-speakleash-avg",
    "score": 0.35164,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-1.5B-Instruct'; Space Average = mean(Sentiment 2.795, Language understanding 1.350, Phraseology 2.225, Tricky questions 0.663) = 1.7582 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.758198,
    "scale_divisor": 5
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "cptub-speakleash-avg",
    "score": 0.333865,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct'; Space Average = mean(Sentiment 1.942, Language understanding 2.116, Phraseology 2.131, Tricky questions 0.489) = 1.6693 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.669326,
    "scale_divisor": 5
  },
  {
    "model": "smollm2-1-7b",
    "bench": "cptub-speakleash-avg",
    "score": 0.299173,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'HuggingFaceTB/SmolLM2-1.7B-Instruct'; Space Average = mean(Sentiment 2.276, Language understanding 1.100, Phraseology 2.355, Tricky questions 0.253) = 1.4959 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.495863,
    "scale_divisor": 5
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "cptub-speakleash-avg",
    "score": 0.280211,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen2.5-0.5B-Instruct'; Space Average = mean(Sentiment 1.955, Language understanding 0.835, Phraseology 2.595, Tricky questions 0.219) = 1.4011 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 1.401057,
    "scale_divisor": 5
  },
  {
    "model": "mistral-small-3",
    "bench": "cptub-speakleash-avg",
    "score": 0.741735,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "source_id": "speakleash-cptub",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Small-24B-Instruct-2501'; Space Average = mean(Sentiment 3.910, Language understanding 3.600, Phraseology 3.875, Tricky questions 3.449) = 3.7087 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.708674,
    "scale_divisor": 5
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "polish-eq-bench-v2",
    "score": 0.4381,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'openchat/openchat-3.5-0106', model path openchat/openchat-3.5-0106; eq-bench_v2_pl score 43.81 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 17:27:10. Latest of 4 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 45.69 (2024-06-18 14:30:24), 45.69 (2024-06-18 15:15:03), 45.69 (2024-06-18 15:22:41).",
    "raw_value": 43.81,
    "scale_divisor": 100
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "polish-eq-bench-v2",
    "score": 0.4626,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'berkeley-nest/Starling-LM-7B-alpha', model path berkeley-nest/Starling-LM-7B-alpha; eq-bench_v2_pl score 46.26 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 14:22:32. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 49.63 (2024-06-18 21:45:18).",
    "raw_value": 46.26,
    "scale_divisor": 100
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "polish-eq-bench-v2",
    "score": 0.451,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Nexusflow/Starling-LM-7B-beta', model path Nexusflow/Starling-LM-7B-beta; eq-bench_v2_pl score 45.1 (divided by 100); 166.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 17:23:18. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 44.91 (2024-06-18 21:51:54).",
    "raw_value": 45.1,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-8b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4627,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'meta-llama/Meta-Llama-3-8B-Instruct', model path meta-llama/Meta-Llama-3-8B-Instruct; eq-bench_v2_pl score 46.27 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 21:06:08. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 46.53 (2024-06-18 22:35:53).",
    "raw_value": 46.27,
    "scale_divisor": 100
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "polish-eq-bench-v2",
    "score": 0.5325,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'mistralai/Mistral-7B-Instruct-v0.2', model path mistralai/Mistral-7B-Instruct-v0.2; eq-bench_v2_pl score 53.25 (divided by 100); 151.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 16:50:36. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 52.99 (2024-06-19 09:49:42).",
    "raw_value": 53.25,
    "scale_divisor": 100
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.5735,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'upstage/SOLAR-10.7B-Instruct-v1.0', model path upstage/SOLAR-10.7B-Instruct-v1.0; eq-bench_v2_pl score 57.35 (divided by 100); 162.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 00:19:48. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 57.57 (2024-06-19 10:43:47).",
    "raw_value": 57.35,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-7b",
    "bench": "polish-eq-bench-v2",
    "score": 0.5374,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen2-7B-Instruct', model path Qwen/Qwen2-7B-Instruct; eq-bench_v2_pl score 53.74 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 18:12:42. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 53.08 (2024-06-19 10:46:52).",
    "raw_value": 53.74,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-1-5b",
    "bench": "polish-eq-bench-v2",
    "score": 0.1533,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen2-1.5B-Instruct', model path Qwen/Qwen2-1.5B-Instruct; eq-bench_v2_pl score 15.33 (divided by 100); 165.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 12:38:29.",
    "raw_value": 15.33,
    "scale_divisor": 100
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "polish-eq-bench-v2",
    "score": 0.1647,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'google/gemma-1.1-2b-it', model path google/gemma-1.1-2b-it; eq-bench_v2_pl score 16.47 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 13:15:24.",
    "raw_value": 16.47,
    "scale_divisor": 100
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.0519,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id '01-ai/Yi-1.5-6B-Chat', model path 01-ai/Yi-1.5-6B-Chat; eq-bench_v2_pl score 5.19 (divided by 100); 161.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 14:11:22.",
    "raw_value": 5.19,
    "scale_divisor": 100
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "polish-eq-bench-v2",
    "score": 0.4521,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'mistralai/Mistral-7B-Instruct-v0.3', model path mistralai/Mistral-7B-Instruct-v0.3; eq-bench_v2_pl score 45.21 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 16:54:49.",
    "raw_value": 45.21,
    "scale_divisor": 100
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.25,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen1.5-7B-Chat', model path Qwen/Qwen1.5-7B-Chat; eq-bench_v2_pl score 25.0 (divided by 100); 164.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 18:03:34.",
    "raw_value": 25,
    "scale_divisor": 100
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "polish-eq-bench-v2",
    "score": 0.3748,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'teknium/OpenHermes-2.5-Mistral-7B', model path teknium/OpenHermes-2.5-Mistral-7B; eq-bench_v2_pl score 37.48 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 20:34:12.",
    "raw_value": 37.48,
    "scale_divisor": 100
  },
  {
    "model": "aya-23-8b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4543,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'CohereForAI/aya-23-8B', model path CohereForAI/aya-23-8B; eq-bench_v2_pl score 45.43 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 20:44:01.",
    "raw_value": 45.43,
    "scale_divisor": 100
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.4878,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id '01-ai/Yi-1.5-9B-Chat', model path 01-ai/Yi-1.5-9B-Chat; eq-bench_v2_pl score 48.78 (divided by 100); 163.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 23:19:16.",
    "raw_value": 48.78,
    "scale_divisor": 100
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.5707,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'microsoft/Phi-3-medium-4k-instruct', model path microsoft/Phi-3-medium-4k-instruct; eq-bench_v2_pl score 57.07 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 02:46:38.",
    "raw_value": 57.07,
    "scale_divisor": 100
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.5126,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen1.5-14B-Chat', model path Qwen/Qwen1.5-14B-Chat; eq-bench_v2_pl score 51.26 (divided by 100); 160.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 02:52:13.",
    "raw_value": 51.26,
    "scale_divisor": 100
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.6069,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen1.5-32B-Chat', model path Qwen/Qwen1.5-32B-Chat; eq-bench_v2_pl score 60.69 (divided by 100); 168.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 13:34:52.",
    "raw_value": 60.69,
    "scale_divisor": 100
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.4632,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id '01-ai/Yi-1.5-34B-Chat', model path 01-ai/Yi-1.5-34B-Chat; eq-bench_v2_pl score 46.32 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 13:51:30.",
    "raw_value": 46.32,
    "scale_divisor": 100
  },
  {
    "model": "aya-23-35b",
    "bench": "polish-eq-bench-v2",
    "score": 0.5841,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'CohereForAI/aya-23-35B', model path CohereForAI/aya-23-35B; eq-bench_v2_pl score 58.41 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 14:03:07.",
    "raw_value": 58.41,
    "scale_divisor": 100
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "polish-eq-bench-v2",
    "score": 0.5864,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'mistralai/Mixtral-8x7B-Instruct-v0.1', model path mistralai/Mixtral-8x7B-Instruct-v0.1; eq-bench_v2_pl score 58.64 (divided by 100); 168.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 14:35:28.",
    "raw_value": 58.64,
    "scale_divisor": 100
  },
  {
    "model": "llama-3-70b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7121,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'meta-llama/Meta-Llama-3-70B-Instruct', model path meta-llama/Meta-Llama-3-70B-Instruct; eq-bench_v2_pl score 71.21 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 17:15:58.",
    "raw_value": 71.21,
    "scale_divisor": 100
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.6803,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen1.5-72B-Chat', model path Qwen/Qwen1.5-72B-Chat; eq-bench_v2_pl score 68.03 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 18:06:58.",
    "raw_value": 68.03,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-72b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7207,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen2-72B-Instruct', model path Qwen/Qwen2-72B-Instruct; eq-bench_v2_pl score 72.07 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 18:55:02.",
    "raw_value": 72.07,
    "scale_divisor": 100
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "polish-eq-bench-v2",
    "score": 0.6763,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2024-06-26",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'mistralai/Mixtral-8x22B-Instruct-v0.1', model path mistralai/Mixtral-8x22B-Instruct-v0.1; eq-bench_v2_pl score 67.63 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-26 23:40:01.",
    "raw_value": 67.63,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "polish-eq-bench-v2",
    "score": 0.2779,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-1.5B-Instruct', model path Qwen/Qwen2.5-1.5B-Instruct; eq-bench_v2_pl score 27.79 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-01 12:09:18.",
    "raw_value": 27.79,
    "scale_divisor": 100
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "polish-eq-bench-v2",
    "score": 0.6179,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'glm-4-9b-chat', model path THUDM/glm-4-9b-chat; eq-bench_v2_pl score 61.79 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-01 12:23:46.",
    "raw_value": 61.79,
    "scale_divisor": 100
  },
  {
    "model": "qwen2-5-3b",
    "bench": "polish-eq-bench-v2",
    "score": 0.3608,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen2.5-3B-Instruct', model path Qwen/Qwen2.5-3B-Instruct; eq-bench_v2_pl score 36.08 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 17:26:57.",
    "raw_value": 36.08,
    "scale_divisor": 100
  },
  {
    "model": "mistral-small-3",
    "bench": "polish-eq-bench-v2",
    "score": 0.7052,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Mistral-Small-24B-Instruct-2501', model path mistralai/Mistral-Small-24B-Instruct-2501; eq-bench_v2_pl score 70.52 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 17:33:14.",
    "raw_value": 70.52,
    "scale_divisor": 100
  },
  {
    "model": "mistral-small-2409",
    "bench": "polish-eq-bench-v2",
    "score": 0.7285,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Mistral-Small-Instruct-2409', model path mistralai/Mistral-Small-Instruct-2409; eq-bench_v2_pl score 72.85 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 17:43:01.",
    "raw_value": 72.85,
    "scale_divisor": 100
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "polish-eq-bench-v2",
    "score": 0.6176,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "source_id": "speakleash-polish-eq-bench",
    "by": "independent",
    "date": "2025-03-03",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Mistral-Nemo-Instruct-2407', model path mistralai/Mistral-Nemo-Instruct-2407; eq-bench_v2_pl score 61.76 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-03 10:29:42.",
    "raw_value": 61.76,
    "scale_divisor": 100
  },
  {
    "model": "gemma-2-9b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3507,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-2-9b-it' (model link https://huggingface.co/google/gemma-2-9b-it); 'Average Performance' 35.07 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "gemma-2-27b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3822,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemma-2-27b-it' (model link https://huggingface.co/google/gemma-2-27b-it); 'Average Performance' 38.22 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1647,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-2-7b-chat' (model link https://huggingface.co/meta-llama/Llama-2-7b-chat-hf); 'Average Performance' 16.47 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2091,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-2-13b-chat' (model link https://huggingface.co/meta-llama/Llama-2-13b-chat-hf); 'Average Performance' 20.91 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2515,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-2-70b-chat' (model link https://huggingface.co/meta-llama/Llama-2-70b-chat-hf); 'Average Performance' 25.15 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3275,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.1-Nemotron-70B-Instruct-HF' (model link https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF); 'Average Performance' 32.75 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3037,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Ministral-8B-Instruct-2410' (model link https://huggingface.co/mistralai/Ministral-8B-Instruct-2410); 'Average Performance' 30.37 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "mistral-small-2409",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3519,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Mistral-Small-Instruct-2409' (model link https://huggingface.co/mistralai/Mistral-Small-Instruct-2409); 'Average Performance' 35.19 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "mistral-small-3",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3753,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Mistral-Small-24B-Instruct-2501' (model link https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501); 'Average Performance' 37.53 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3973,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Mistral-Small-3.1-24B-Instruct-2503' (model link https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503); 'Average Performance' 39.73 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "magistral-small-2506",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3756,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'Magistral-Small-2506' (model link https://huggingface.co/mistralai/Magistral-Small-2506); 'Average Performance' 37.56 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2541,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Phi-3.5-mini-instruct' (model link https://huggingface.co/microsoft/Phi-3.5-mini-instruct); 'Average Performance' 25.41 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2954,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Phi-3.5-MoE-instruct' (model link https://huggingface.co/microsoft/Phi-3.5-MoE-instruct); 'Average Performance' 29.54 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2216,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen2.5-1.5B-Instruct' (model link https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct); 'Average Performance' 22.16 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "qwen2-5-3b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2659,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen2.5-3B-Instruct' (model link https://huggingface.co/Qwen/Qwen2.5-3B-Instruct); 'Average Performance' 26.59 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "openthinker3-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1979,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'OpenThinker3-7B' (model link https://huggingface.co/open-thoughts/OpenThinker3-7B); 'Average Performance' 19.79 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "athene-v2-chat",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4169,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "source_id": "bridge-medical-leaderboard",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Athene-V2-Chat' (model link https://huggingface.co/Nexusflow/Athene-V2-Chat); 'Average Performance' 41.69 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38."
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "cruxeval-zeroeval",
    "score": 0.4475,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-9B-Chat'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=1.62. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "cruxeval-zeroeval",
    "score": 0.4412,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-34B-Chat'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "cruxeval-zeroeval",
    "score": 0.4212,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Phi-3.5-mini-instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=3.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-7b",
    "bench": "cruxeval-zeroeval",
    "score": 0.3788,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-7B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-5-3b",
    "bench": "cruxeval-zeroeval",
    "score": 0.3312,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2.5-3B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=1.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "gemma-2-2b",
    "bench": "cruxeval-zeroeval",
    "score": 0.215,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'gemma-2-2b-it'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "gsm8k-zeroeval",
    "score": 0.8408,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-34B-Chat'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "gsm8k-zeroeval",
    "score": 0.8279,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Mistral-Nemo-Instruct-2407'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "gsm8k-zeroeval",
    "score": 0.8203,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Phi-3.5-mini-instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=1.21. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-7b",
    "bench": "gsm8k-zeroeval",
    "score": 0.8006,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-7B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "gsm8k-zeroeval",
    "score": 0.7642,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-9B-Chat'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "gemma-2-2b",
    "bench": "gsm8k-zeroeval",
    "score": 0.5163,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'gemma-2-2b-it'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.38. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-1-5b",
    "bench": "gsm8k-zeroeval",
    "score": 0.4337,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-1.5B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=4.78. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7279,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-34B-Chat'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.01. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6767,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Phi-3.5-mini-instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=4.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-7b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6692,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-7B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.72. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6688,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Mistral-Nemo-Instruct-2407'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.47. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6505,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-9B-Chat'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=4.61. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-5-3b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6425,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2.5-3B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.72. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "gemma-2-2b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.5194,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'gemma-2-2b-it'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=2.59. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-1-5b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.4111,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-1.5B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=7.74. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "zebralogic-zeroeval",
    "score": 0.118,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Mistral-Nemo-Instruct-2407'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.60. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "zebralogic-zeroeval",
    "score": 0.115,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-34B-Chat'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=4.40. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "mathstral-7b-v0-1",
    "bench": "zebralogic-zeroeval",
    "score": 0.09,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'mathstral-7B-v0.1'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=36.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-7b",
    "bench": "zebralogic-zeroeval",
    "score": 0.084,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2-7B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=24.40. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "zebralogic-zeroeval",
    "score": 0.064,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Phi-3.5-mini-instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=80.60. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "qwen2-5-3b",
    "bench": "zebralogic-zeroeval",
    "score": 0.048,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Qwen2.5-3B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=56.70. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "gemma-2-2b",
    "bench": "zebralogic-zeroeval",
    "score": 0.042,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'gemma-2-2b-it'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=57.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "zebralogic-zeroeval",
    "score": 0.023,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "source_id": "zeroeval-published-results",
    "note": "Published entry 'Yi-1.5-9B-Chat'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=11.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "bfcl-v4",
    "score": 0.2763,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Open-Mistral-Nemo-2407 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://mistral.ai/news/mistral-nemo/. Served through Mistral's API id open-mistral-nemo-2407, Mistral's endpoint for the open Mistral NeMo 12B Instruct 2407 weights."
  },
  {
    "model": "granite-3-1-8b",
    "bench": "bfcl-v4",
    "score": 0.271,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Granite-3.1-8B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/ibm-granite/granite-3.1-8b-instruct."
  },
  {
    "model": "falcon3-10b",
    "bench": "bfcl-v4",
    "score": 0.2701,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Falcon3-10B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/tiiuae/Falcon3-10B-Instruct."
  },
  {
    "model": "granite-3-2-8b",
    "bench": "bfcl-v4",
    "score": 0.2687,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Granite-3.2-8B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/ibm-granite/granite-3.2-8b-instruct."
  },
  {
    "model": "falcon3-7b",
    "bench": "bfcl-v4",
    "score": 0.2403,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Falcon3-7B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/tiiuae/Falcon3-7B-Instruct."
  },
  {
    "model": "granite-4-0-350m",
    "bench": "bfcl-v4",
    "score": 0.1898,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Granite-4.0-350m (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/ibm-granite/granite-4.0-350m."
  },
  {
    "model": "falcon3-3b",
    "bench": "bfcl-v4",
    "score": 0.1625,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Falcon3-3B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/tiiuae/Falcon3-3B-Instruct."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "bfcl-v4",
    "score": 0.111,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Ministral-8B-Instruct-2410 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/mistralai/Ministral-8B-Instruct-2410.",
    "invalid": "Same pattern as Nemotron Ultra: zero on every category that needs a call and 100% on irrelevance, so the FC handler never got a parseable call."
  },
  {
    "model": "falcon3-1b",
    "bench": "bfcl-v4",
    "score": 0.1108,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4",
    "note": "Published entry 'Falcon3-1B-Instruct (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model link in the archive: https://huggingface.co/tiiuae/Falcon3-1B-Instruct."
  },
  {
    "model": "mistral-small-3",
    "bench": "aragen-v3-3c3h",
    "score": 0.3984,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-25",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Mistral-Small-24B-Instruct-2501'; 3C3H Score 0.3984 (95% CI 0.355753-0.441449, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250925_090225; RM_Thinking_Traces=None."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "aragen-v3-3c3h",
    "score": 0.1761,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-26",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Ministral-8B-Instruct-2410'; 3C3H Score 0.1761 (95% CI 0.140304-0.213477, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250926_091211; RM_Thinking_Traces=None."
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "aragen-v3-3c3h",
    "score": 0.0508,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-27",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Mistral-7B-Instruct-v0.3'; 3C3H Score 0.0508 (95% CI 0.031807-0.071717, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250927_052058; RM_Thinking_Traces=None."
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "aragen-v3-3c3h",
    "score": 0.0048,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-27",
    "note": "Inception AraGen-v3 leaderboard entry 'mistralai/Mistral-7B-Instruct-v0.2'; 3C3H Score 0.0048 (95% CI 0.0-0.012088, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250927_074922; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-70b",
    "bench": "aragen-v3-3c3h",
    "score": 0.2614,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Meta-Llama-3-70B-Instruct'; 3C3H Score 0.2614 (95% CI 0.21978-0.304441, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_161911; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-3-8b",
    "bench": "aragen-v3-3c3h",
    "score": 0.0921,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Meta-Llama-3-8B-Instruct'; 3C3H Score 0.0921 (95% CI 0.065799-0.1211, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250930_011310; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "aragen-v3-3c3h",
    "score": 0.1322,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen2.5-1.5B-Instruct'; 3C3H Score 0.1322 (95% CI 0.111024-0.151792, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251001_031053; RM_Thinking_Traces=None."
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "aragen-v3-3c3h",
    "score": 0.0521,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'Qwen/Qwen2.5-0.5B-Instruct'; 3C3H Score 0.0521 (95% CI 0.033823-0.071117, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251001_052625; RM_Thinking_Traces=None."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.02,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-2-13b-chat-hf'; 'Ar Prompt-lvl' 0.02 (Ar instruction-level 0.025, En prompt-level 0.325 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.007,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-2-7b-chat-hf'; 'Ar Prompt-lvl' 0.007 (Ar instruction-level 0.013, En prompt-level 0.322 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.05,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen1.5-14B-Chat'; 'Ar Prompt-lvl' 0.05 (Ar instruction-level 0.055, En prompt-level 0.498 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.141,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen1.5-72B-Chat'; 'Ar Prompt-lvl' 0.141 (Ar instruction-level 0.152, En prompt-level 0.49 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-7b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.381,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2-7B-Instruct'; 'Ar Prompt-lvl' 0.381 (Ar instruction-level 0.46, En prompt-level 0.516 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.139,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2.5-0.5B-Instruct'; 'Ar Prompt-lvl' 0.139 (Ar instruction-level 0.207, En prompt-level 0.251 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "qwen2-5-3b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.381,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'Qwen/Qwen2.5-3B-Instruct'; 'Ar Prompt-lvl' 0.381 (Ar instruction-level 0.473, En prompt-level 0.561 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.03,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'meta-llama/Llama-2-70b-chat-hf'; 'Ar Prompt-lvl' 0.03 (Ar instruction-level 0.035, En prompt-level 0.422 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "gemma-2-9b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.515,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'google/gemma-2-9b-it'; 'Ar Prompt-lvl' 0.515 (Ar instruction-level 0.606, En prompt-level 0.696 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "gemma-2-27b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.594,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'google/gemma-2-27b-it'; 'Ar Prompt-lvl' 0.594 (Ar instruction-level 0.677, En prompt-level 0.761 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "aya-expanse-8b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.485,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'CohereForAI/aya-expanse-8b'; 'Ar Prompt-lvl' 0.485 (Ar instruction-level 0.585, En prompt-level 0.578 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "aya-expanse-32b",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.567,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "source_id": "inception-leaderboards-space",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'CohereForAI/aya-expanse-32b'; 'Ar Prompt-lvl' 0.567 (Ar instruction-level 0.66, En prompt-level 0.651 not imported). No run date in the file; date is the pinned Space revision date."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "click-korean-kim",
    "score": 0.4299,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-mini-instruct'. microsoft/Phi-3.5-mini-instruct; source label Phi-3.5-mini-instruct. Personal independent run; CLIcK, Overall accuracy, zero-shot (open-source models table, line 48 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "click-korean-kim",
    "score": 0.5644,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-MoE-instruct'. microsoft/Phi-3.5-MoE-instruct; source label Phi-3.5-MoE-instruct. Personal independent run; CLIcK, Overall accuracy, zero-shot (open-source models table, line 48 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "haerae-korean-kim",
    "score": 0.4421,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-mini-instruct'. microsoft/Phi-3.5-mini-instruct; source label Phi-3.5-mini-instruct. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot (open-source models table, line 93 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "haerae-korean-kim",
    "score": 0.6183,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-MoE-instruct'. microsoft/Phi-3.5-MoE-instruct; source label Phi-3.5-MoE-instruct. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot (open-source models table, line 93 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "kmmlu-hard-kim",
    "score": 0.24,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-mini-instruct'. microsoft/Phi-3.5-mini-instruct; source label Phi-3.5-mini-instruct. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot (open-source models table, line 244 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "kmmlu-hard-kim",
    "score": 0.2534,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "source_id": "korean-personal-benchmark-runs",
    "note": "Published entry 'Phi-3.5-MoE-instruct'. microsoft/Phi-3.5-MoE-instruct; source label Phi-3.5-MoE-instruct. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot (open-source models table, line 244 of the pinned file). As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. Table-only evidence: no matching raw CSV released in the pinned repository."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.912,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'mistral-small-2503'; column 'Model Avg' = 0.912; single-turn 0.93, multi-turn 0.89; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Mistral API id 'mistral-small-2503' serves the open-weight Mistral-Small-3.1-24B-Instruct-2503 checkpoint."
  },
  {
    "model": "mistral-small-2409",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.75,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'mistral-small-2409'; column 'Model Avg' = 0.750; single-turn 0.82, multi-turn 0.68; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Mistral API id 'mistral-small-2409' serves the open-weight Mistral-Small-Instruct-2409 checkpoint."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.689,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'ministral-8b-2410'; column 'Model Avg' = 0.689; single-turn 0.73, multi-turn 0.65; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Mistral API id 'ministral-8b-2410' serves the open-weight Ministral-8B-Instruct-2410 checkpoint."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.661,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'open-mistral-nemo-2407'; column 'Model Avg' = 0.661; single-turn 0.68, multi-turn 0.64; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Mistral API id 'open-mistral-nemo-2407' serves the open-weight Mistral-Nemo-Instruct-2407 checkpoint."
  },
  {
    "model": "magistral-small-2506",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.16,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "source_id": "galileo-agent-leaderboard",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'magistral-small-2506'; column 'Avg AC' = 0.16; Avg TSQ 0.53; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric. Mistral API id 'magistral-small-2506' serves the open-weight Magistral-Small-2506 checkpoint."
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.065,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-1.3b-Ins'; model_name=deepseek-ai/deepseek-coder-1.3b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=6.4857881137%; website rounding=6.5%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.179,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-6.7b-Ins'; model_name=deepseek-ai/deepseek-coder-6.7b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=17.8811369509%; website rounding=17.9%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.219,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-33b-Ins'; model_name=deepseek-ai/deepseek-coder-33b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=21.8604651163%; website rounding=21.9%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.206,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-Coder-V2-Lite-Instruct'; model_name=deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct; link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=20.5684754522%; website rounding=20.6%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.036,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-V2-Lite-Chat'; model_name=deepseek-ai/DeepSeek-V2-Lite-Chat; link https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=3.5917312661%; website rounding=3.6%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "llama-3-8b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.133,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3-8b-Ins'; model_name=meta-llama/Meta-Llama-3-8B-Instruct; link https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=13.3333333333%; website rounding=13.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "llama-3-70b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.243,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'LLama3-70b-Ins'; model_name=meta-llama/Meta-Llama-3-70B-Instruct; link https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=24.3152454780%; website rounding=24.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "qwen2-72b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.253,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Qwen2-Ins-72B'; model_name=Qwen/Qwen2-72B-Instruct; link https://huggingface.co/Qwen/Qwen2-72B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.3488372093%; website rounding=25.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.235,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "source_id": "livecodebench-website-20241226",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Yi-Coder-9B-Chat'; model_name=01-ai/Yi-Coder-9B-Chat; link https://huggingface.co/01-ai/Yi-Coder-9B-Chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=23.5400516796%; website rounding=23.5%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance."
  },
  {
    "model": "granite-3-1-2b",
    "bench": "arena-hard-unspecified",
    "score": 0.233,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'Arena-Hard' = 23.3 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "mmlu",
    "score": 0.5711,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'MMLU' = 57.11 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "popqa",
    "score": 0.2055,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'PopQA' = 20.55 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "bbh",
    "score": 0.6182,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'BigBenchHard' = 61.82 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "drop",
    "score": 0.2099,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'DROP' = 20.99 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "gsm8k",
    "score": 0.6755,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'GSM8K' = 67.55 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "ifeval-unspecified",
    "score": 0.6359,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-2B-Instruct', column 'IFEval' = 63.59 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "arena-hard-unspecified",
    "score": 0.2486,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'Arena-Hard' = 24.86 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "mmlu",
    "score": 0.5718,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'MMLU' = 57.18 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "popqa",
    "score": 0.2056,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'PopQA' = 20.56 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "bbh",
    "score": 0.6139,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'BigBenchHard' = 61.39 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "drop",
    "score": 0.2384,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'DROP' = 23.84 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "gsm8k",
    "score": 0.6702,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'GSM8K' = 67.02 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "ifeval-unspecified",
    "score": 0.6155,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-2B-Instruct', column 'IFEval' = 61.55 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "arena-hard-unspecified",
    "score": 0.2886,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'Arena-Hard' = 28.86 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "mmlu",
    "score": 0.5588,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'MMLU' = 55.88 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "popqa",
    "score": 0.184,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'PopQA' = 18.4 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "bbh",
    "score": 0.6391,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'BigBenchHard' = 63.91 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "drop",
    "score": 0.4433,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'DROP' = 44.33 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "gsm8k",
    "score": 0.7248,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'GSM8K' = 72.48 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "ifeval-unspecified",
    "score": 0.658,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-2B-Instruct', column 'IFEval' = 65.8 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "arena-hard-unspecified",
    "score": 0.3758,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'Arena-Hard' = 37.58 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "mmlu",
    "score": 0.6677,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'MMLU' = 66.77 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "popqa",
    "score": 0.287,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'PopQA' = 28.7 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "bbh",
    "score": 0.6987,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'BigBenchHard' = 69.87 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "drop",
    "score": 0.5857,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'DROP' = 58.57 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "gsm8k",
    "score": 0.7915,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'GSM8K' = 79.15 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "ifeval-unspecified",
    "score": 0.732,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.1-8B-Instruct', column 'IFEval' = 73.20 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "arena-hard-unspecified",
    "score": 0.5525,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'Arena-Hard' = 55.25 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "mmlu",
    "score": 0.6679,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'MMLU' = 66.79 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "popqa",
    "score": 0.2804,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'PopQA' = 28.04 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "bbh",
    "score": 0.7186,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'BigBenchHard' = 71.86 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "drop",
    "score": 0.5829,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'DROP' = 58.29 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "gsm8k",
    "score": 0.8165,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'GSM8K' = 81.65 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "ifeval-unspecified",
    "score": 0.7431,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.2-8B-Instruct', column 'IFEval' = 74.31 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "arena-hard-unspecified",
    "score": 0.5756,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'Arena-Hard' = 57.56 (divided by 100). Arena-Hard (version/judge not stated; footnote 1: not an OLMES score). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "mmlu",
    "score": 0.6554,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'MMLU' = 65.54 (divided by 100). MMLU (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "popqa",
    "score": 0.2617,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'PopQA' = 26.17 (divided by 100). PopQA (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "bbh",
    "score": 0.6913,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'BigBenchHard' = 69.13 (divided by 100). BigBenchHard (OLMES; footnote 2: regex added for answer extraction). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "drop",
    "score": 0.5936,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'DROP' = 59.36 (divided by 100). DROP (OLMES; footnote 3: implementation modified). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "gsm8k",
    "score": 0.8089,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'GSM8K' = 80.89 (divided by 100). GSM8K (OLMES). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "ifeval-unspecified",
    "score": 0.7482,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 1; row 'Granite-3.3-8B-Instruct', column 'IFEval' = 74.82 (divided by 100). IFEval (OLMES; prompt/instruction and strict/loose not stated in the card). Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "aime-2024",
    "score": 0.0089,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.1-2B-Instruct', column 'AIME24' = 0.89 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "math-500",
    "score": 0.3507,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.1-2B-Instruct', column 'MATH-500' = 35.07 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "aime-2024",
    "score": 0.0089,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.2-2B-Instruct', column 'AIME24' = 0.89 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "math-500",
    "score": 0.3554,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.2-2B-Instruct', column 'MATH-500' = 35.54 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "aime-2024",
    "score": 0.0328,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.3-2B-Instruct', column 'AIME24' = 3.28 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-2b",
    "bench": "math-500",
    "score": 0.5809,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.3-2B-Instruct', column 'MATH-500' = 58.09 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "aime-2024",
    "score": 0.0197,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.1-8B-Instruct', column 'AIME24' = 1.97 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "math-500",
    "score": 0.4873,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.1-8B-Instruct', column 'MATH-500' = 48.73 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "aime-2024",
    "score": 0.0243,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.2-8B-Instruct', column 'AIME24' = 2.43 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "math-500",
    "score": 0.528,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.2-8B-Instruct', column 'MATH-500' = 52.8 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "aime-2024",
    "score": 0.0812,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.3-8B-Instruct', column 'AIME24' = 8.12 (divided by 100). AIME 2024, pass@1 as reported. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "math-500",
    "score": 0.6902,
    "source": "https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/51dd4bc2ade4059a6bd87649d68aa11e4fb2529b/README.md",
    "by": "lab",
    "date": "2025-05-12",
    "note": "IBM granite-3.3-8b-instruct model card @51dd4bc, 'Evaluation Results' table 2; row 'Granite-3.3-8B-Instruct', column 'MATH-500' = 69.02 (divided by 100). MATH-500. Card footnote 1: 'Evaluated using OLMES (except AttaQ and Arena-Hard scores)'. Thinking mode not stated for the table. Own Granite rows only; competitor rows, AlpacaEval-2.0, TruthfulQA and HumanEval/HumanEval+ (metric variant not stated) and AttaQ not imported.",
    "source_id": "card2-ibm-granite-granite-3-3-8b-instruct"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "alpacaeval2-lc",
    "score": 0.387,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'AE2' = 38.7 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "bbh",
    "score": 0.666,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'BBH' = 66.6 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "gsm8k",
    "score": 0.743,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'GSM8K' = 74.3 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "ifeval-prompt-loose",
    "score": 0.669,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'IFE' = 66.9 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "math-subset-unspecified",
    "score": 0.412,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'MATH' = 41.2 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "mmlu",
    "score": 0.702,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'MMLU' = 70.2 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "popqa",
    "score": 0.45,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'PQA' = 45.0 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "truthfulqa-mc2",
    "score": 0.629,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT-3.5 Turbo 0125', column 'TQA' = 62.9 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "alpacaeval2-lc",
    "score": 0.497,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'AE2' = 49.7 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "bbh",
    "score": 0.659,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'BBH' = 65.9 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "gsm8k",
    "score": 0.83,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'GSM8K' = 83.0 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "ifeval-prompt-loose",
    "score": 0.835,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'IFE' = 83.5 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "math-subset-unspecified",
    "score": 0.679,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'MATH' = 67.9 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu",
    "score": 0.822,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'MMLU' = 82.2 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "popqa",
    "score": 0.39,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'PQA' = 39.0 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "truthfulqa-mc2",
    "score": 0.648,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'GPT 4o Mini 0724', column 'TQA' = 64.8 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "alpacaeval2-lc",
    "score": 0.204,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'AE2' = 20.4 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "bbh",
    "score": 0.394,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'BBH' = 39.4 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gsm8k",
    "score": 0.35,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'GSM8K' = 35.0 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifeval-prompt-loose",
    "score": 0.606,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'IFE' = 60.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "math-subset-unspecified",
    "score": 0.403,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'MATH' = 40.3 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmlu",
    "score": 0.389,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'MMLU' = 38.9 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "popqa",
    "score": 0.096,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'PQA' = 9.6 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-1b",
    "bench": "truthfulqa-mc2",
    "score": 0.438,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 1B', column 'TQA' = 43.8 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "alpacaeval2-lc",
    "score": 0.101,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'AE2' = 10.1 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bbh",
    "score": 0.402,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'BBH' = 40.2 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gsm8k",
    "score": 0.454,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'GSM8K' = 45.4 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifeval-prompt-loose",
    "score": 0.54,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'IFE' = 54.0 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "math-subset-unspecified",
    "score": 0.216,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'MATH' = 21.6 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmlu",
    "score": 0.467,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'MMLU' = 46.7 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "popqa",
    "score": 0.138,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'PQA' = 13.8 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "truthfulqa-mc2",
    "score": 0.415,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.2 1B', column 'TQA' = 41.5 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "alpacaeval2-lc",
    "score": 0.074,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'AE2' = 7.4 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "bbh",
    "score": 0.458,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'BBH' = 45.8 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "gsm8k",
    "score": 0.662,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'GSM8K' = 66.2 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "ifeval-prompt-loose",
    "score": 0.442,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'IFE' = 44.2 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "math-subset-unspecified",
    "score": 0.406,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'MATH' = 40.6 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "mmlu",
    "score": 0.597,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'MMLU' = 59.7 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "popqa",
    "score": 0.155,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'PQA' = 15.5 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "truthfulqa-mc2",
    "score": 0.465,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 1.5B', column 'TQA' = 46.5 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "alpacaeval2-lc",
    "score": 0.314,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'AE2' = 31.4 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "bbh",
    "score": 0.708,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'BBH' = 70.8 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "gsm8k",
    "score": 0.8,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'GSM8K' = 80.0 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "ifeval-prompt-loose",
    "score": 0.564,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'IFE' = 56.4 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "math-subset-unspecified",
    "score": 0.4,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'MATH' = 40.0 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "mmlu",
    "score": 0.685,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'MMLU' = 68.5 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "popqa",
    "score": 0.202,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'PQA' = 20.2 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "truthfulqa-mc2",
    "score": 0.555,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Ministral 8B 2410', column 'TQA' = 55.5 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "alpacaeval2-lc",
    "score": 0.258,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'AE2' = 25.8 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bbh",
    "score": 0.719,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'BBH' = 71.9 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gsm8k",
    "score": 0.834,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'GSM8K' = 83.4 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "ifeval-prompt-loose",
    "score": 0.806,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'IFE' = 80.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "math-subset-unspecified",
    "score": 0.425,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'MATH' = 42.5 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu",
    "score": 0.713,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'MMLU' = 71.3 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "popqa",
    "score": 0.284,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'PQA' = 28.4 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "truthfulqa-mc2",
    "score": 0.551,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 8B', column 'TQA' = 55.1 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "alpacaeval2-lc",
    "score": 0.34,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'AE2' = 34.0 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "bbh",
    "score": 0.69,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'BBH' = 69.0 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "gsm8k",
    "score": 0.876,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'GSM8K' = 87.6 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "ifeval-prompt-loose",
    "score": 0.824,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'IFE' = 82.4 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.437,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'MATH' = 43.7 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "mmlu",
    "score": 0.682,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'MMLU' = 68.2 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "popqa",
    "score": 0.291,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'PQA' = 29.1 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "truthfulqa-mc2",
    "score": 0.55,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Tulu 3 8B', column 'TQA' = 55.0 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.297,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'AE2' = 29.7 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "bbh",
    "score": 0.702,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'BBH' = 70.2 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "gsm8k",
    "score": 0.838,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'GSM8K' = 83.8 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "ifeval-prompt-loose",
    "score": 0.747,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'IFE' = 74.7 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "math-subset-unspecified",
    "score": 0.699,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'MATH' = 69.9 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "mmlu",
    "score": 0.766,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'MMLU' = 76.6 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "popqa",
    "score": 0.181,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'PQA' = 18.1 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.631,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 7B', column 'TQA' = 63.1 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "alpacaeval2-lc",
    "score": 0.437,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'AE2' = 43.7 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "bbh",
    "score": 0.649,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'BBH' = 64.9 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gsm8k",
    "score": 0.797,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'GSM8K' = 79.7 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "ifeval-prompt-loose",
    "score": 0.699,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'IFE' = 69.9 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "math-subset-unspecified",
    "score": 0.298,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'MATH' = 29.8 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mmlu",
    "score": 0.691,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'MMLU' = 69.1 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "popqa",
    "score": 0.283,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'PQA' = 28.3 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-9b",
    "bench": "truthfulqa-mc2",
    "score": 0.614,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 9B', column 'TQA' = 61.4 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "alpacaeval2-lc",
    "score": 0.346,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'AE2' = 34.6 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "bbh",
    "score": 0.784,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'BBH' = 78.4 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "gsm8k",
    "score": 0.839,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'GSM8K' = 83.9 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "ifeval-prompt-loose",
    "score": 0.824,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'IFE' = 82.4 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "math-subset-unspecified",
    "score": 0.706,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'MATH' = 70.6 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "mmlu",
    "score": 0.811,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'MMLU' = 81.1 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "popqa",
    "score": 0.211,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'PQA' = 21.1 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "truthfulqa-mc2",
    "score": 0.708,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 14B', column 'TQA' = 70.8 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "alpacaeval2-lc",
    "score": 0.49,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'AE2' = 49.0 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "bbh",
    "score": 0.727,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'BBH' = 72.7 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "gsm8k",
    "score": 0.807,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'GSM8K' = 80.7 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "ifeval-prompt-loose",
    "score": 0.632,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'IFE' = 63.2 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "math-subset-unspecified",
    "score": 0.351,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'MATH' = 35.1 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "mmlu",
    "score": 0.707,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'MMLU' = 70.7 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "popqa",
    "score": 0.339,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'PQA' = 33.9 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-2-27b",
    "bench": "truthfulqa-mc2",
    "score": 0.646,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 2 27B', column 'TQA' = 64.6 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "alpacaeval2-lc",
    "score": 0.391,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'AE2' = 39.1 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "bbh",
    "score": 0.823,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'BBH' = 82.3 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "gsm8k",
    "score": 0.875,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'GSM8K' = 87.5 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "ifeval-prompt-loose",
    "score": 0.824,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'IFE' = 82.4 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "math-subset-unspecified",
    "score": 0.779,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'MATH' = 77.9 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "mmlu",
    "score": 0.847,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'MMLU' = 84.7 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "popqa",
    "score": 0.261,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'PQA' = 26.1 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "truthfulqa-mc2",
    "score": 0.706,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 32B', column 'TQA' = 70.6 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "alpacaeval2-lc",
    "score": 0.824,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'AE2' = 82.4 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "bbh",
    "score": 0.896,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'BBH' = 89.6 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "gsm8k",
    "score": 0.955,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'GSM8K' = 95.5 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "ifeval-prompt-loose",
    "score": 0.858,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'IFE' = 85.8 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "math-subset-unspecified",
    "score": 0.981,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'MATH' = 98.1 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwq-32b",
    "bench": "mmlu",
    "score": 0.884,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen QwQ 32B', column 'MMLU' = 88.4 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). QwQ-32B run by the authors with its recommended sampling (32K context, temperature 0.6, top-p 0.95, top-k 30), thinking tokens removed before grading. Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "alpacaeval2-lc",
    "score": 0.634,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'AE2' = 63.4 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "bbh",
    "score": 0.837,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'BBH' = 83.7 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "gsm8k",
    "score": 0.911,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'GSM8K' = 91.1 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "ifeval-prompt-loose",
    "score": 0.834,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'IFE' = 83.4 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "math-subset-unspecified",
    "score": 0.762,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'MATH' = 76.2 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "mmlu",
    "score": 0.818,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'MMLU' = 81.8 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "popqa",
    "score": 0.309,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'PQA' = 30.9 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "gemma-3-27b",
    "bench": "truthfulqa-mc2",
    "score": 0.639,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Gemma 3 27B', column 'TQA' = 63.9 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "alpacaeval2-lc",
    "score": 0.477,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'AE2' = 47.7 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "bbh",
    "score": 0.804,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'BBH' = 80.4 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "gsm8k",
    "score": 0.895,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'GSM8K' = 89.5 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "ifeval-prompt-loose",
    "score": 0.876,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'IFE' = 87.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "math-subset-unspecified",
    "score": 0.759,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'MATH' = 75.9 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "mmlu",
    "score": 0.855,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'MMLU' = 85.5 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "popqa",
    "score": 0.306,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'PQA' = 30.6 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "truthfulqa-mc2",
    "score": 0.699,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Qwen 2.5 72B', column 'TQA' = 69.9 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.329,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'AE2' = 32.9 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "bbh",
    "score": 0.83,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'BBH' = 83.0 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "gsm8k",
    "score": 0.945,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'GSM8K' = 94.5 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "ifeval-prompt-loose",
    "score": 0.88,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'IFE' = 88.0 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "math-subset-unspecified",
    "score": 0.562,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'MATH' = 56.2 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "mmlu",
    "score": 0.852,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'MMLU' = 85.2 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "popqa",
    "score": 0.465,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'PQA' = 46.5 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "truthfulqa-mc2",
    "score": 0.668,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.1 70B', column 'TQA' = 66.8 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.365,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'AE2' = 36.5 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bbh",
    "score": 0.858,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'BBH' = 85.8 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "gsm8k",
    "score": 0.936,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'GSM8K' = 93.6 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "ifeval-prompt-loose",
    "score": 0.908,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'IFE' = 90.8 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "math-subset-unspecified",
    "score": 0.718,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'MATH' = 71.8 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "mmlu",
    "score": 0.859,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'MMLU' = 85.9 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "popqa",
    "score": 0.482,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'PQA' = 48.2 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "truthfulqa-mc2",
    "score": 0.661,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'Llama 3.3 70B', column 'TQA' = 66.1 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.058,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'AE2' = 5.8 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "bbh",
    "score": 0.398,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'BBH' = 39.8 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "gsm8k",
    "score": 0.453,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'GSM8K' = 45.3 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "ifeval-prompt-loose",
    "score": 0.516,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'IFE' = 51.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "math-subset-unspecified",
    "score": 0.203,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'MATH' = 20.3 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "mmlu",
    "score": 0.343,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'MMLU' = 34.3 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "popqa",
    "score": 0.164,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'PQA' = 16.4 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.453,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "independent",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'SmolLM2 1.7B', column 'TQA' = 45.3 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "alpacaeval2-lc",
    "score": 0.091,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'AE2' = 9.1 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "bbh",
    "score": 0.35,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'BBH' = 35.0 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "gsm8k",
    "score": 0.683,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'GSM8K' = 68.3 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "ifeval-prompt-loose",
    "score": 0.701,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'IFE' = 70.1 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "math-subset-unspecified",
    "score": 0.207,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'MATH' = 20.7 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "mmlu",
    "score": 0.4,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'MMLU' = 40.0 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "popqa",
    "score": 0.129,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'PQA' = 12.9 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-1b",
    "bench": "truthfulqa-mc2",
    "score": 0.487,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 1B', column 'TQA' = 48.7 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.291,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'AE2' = 29.1 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "bbh",
    "score": 0.514,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'BBH' = 51.4 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "gsm8k",
    "score": 0.851,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'GSM8K' = 85.1 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "ifeval-prompt-loose",
    "score": 0.723,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'IFE' = 72.3 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "math-subset-unspecified",
    "score": 0.325,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'MATH' = 32.5 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "mmlu",
    "score": 0.613,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'MMLU' = 61.3 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "popqa",
    "score": 0.232,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'PQA' = 23.2 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.565,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 7B', column 'TQA' = 56.5 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "alpacaeval2-lc",
    "score": 0.395,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'AE2' = 39.5 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "bbh",
    "score": 0.63,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'BBH' = 63.0 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "gsm8k",
    "score": 0.874,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'GSM8K' = 87.4 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "ifeval-prompt-loose",
    "score": 0.826,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'IFE' = 82.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "math-subset-unspecified",
    "score": 0.392,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'MATH' = 39.2 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "mmlu",
    "score": 0.685,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'MMLU' = 68.5 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "popqa",
    "score": 0.288,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'PQA' = 28.8 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-13b",
    "bench": "truthfulqa-mc2",
    "score": 0.643,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 13B', column 'TQA' = 64.3 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "alpacaeval2-lc",
    "score": 0.428,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'AE2' = 42.8 (divided by 100). AlpacaEval 2, LC win rate, 0-shot, chat template, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "bbh",
    "score": 0.706,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'BBH' = 70.6 (divided by 100). BigBenchHard, 3-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "gsm8k",
    "score": 0.876,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'GSM8K' = 87.6 (divided by 100). GSM8K, 8-shot CoT, EM, multiturn ICL, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "ifeval-prompt-loose",
    "score": 0.856,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'IFE' = 85.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "math-subset-unspecified",
    "score": 0.497,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'MATH' = 49.7 (divided by 100). MATH, 4-shot CoT, Flex EM (OLMES task; the paper does not state the problem subset), from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "mmlu",
    "score": 0.773,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'MMLU' = 77.3 (divided by 100). MMLU, 0-shot CoT ('summarize reasoning' prompt), EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "popqa",
    "score": 0.375,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'PQA' = 37.5 (divided by 100). PopQA, 15-shot, EM, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "olmo-2-32b",
    "bench": "truthfulqa-mc2",
    "score": 0.732,
    "source": "https://arxiv.org/html/2501.00656v3",
    "by": "lab",
    "date": "2025-10-08",
    "note": "OLMo 2 paper (Team OLMo, arXiv 2501.00656v3), Table 7 'OLMo 2 Instruct ... relative to peer open weight models' (caption: 'All models in this table are the instruction tuned variants'); row 'OLMo 2 32B', column 'TQA' = 73.2 (divided by 100). TruthfulQA, 6-shot, MC2, from the Tülu 3 / OLMES instruct evaluation suite (Table 21). Evaluated by Ai2 with that suite (text: 'evaluated in Table 7 ... from the same evaluation suite used by Lambert et al. (2024)'). Average, FLOP, Safety and DROP columns not imported (DROP: format-sensitive, e.g. Qwen 2.5 72B 34.2 vs 7B 54.4; the Tülu 3 paper notes many low values come from few-shot formatting failures).",
    "source_id": "paper-2501.00656v3"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "ifeval-unspecified",
    "score": 0.567,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'IFEval (Average)', column 'SmolLM2-1.7B' = 56.7 (divided by 100). IFEval, average of prompt- and instruction-level (strictness not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifeval-unspecified",
    "score": 0.535,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'IFEval (Average)', column 'Llama3.2-1B' = 53.5 (divided by 100). IFEval, average of prompt- and instruction-level (strictness not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "ifeval-unspecified",
    "score": 0.474,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'IFEval (Average)', column 'Qwen2.5-1.5B' = 47.4 (divided by 100). IFEval, average of prompt- and instruction-level (strictness not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "bbh",
    "score": 0.322,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'BBH (3-shot)', column 'SmolLM2-1.7B' = 32.2 (divided by 100). BBH 3-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bbh",
    "score": 0.276,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'BBH (3-shot)', column 'Llama3.2-1B' = 27.6 (divided by 100). BBH 3-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "bbh",
    "score": 0.353,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'BBH (3-shot)', column 'Qwen2.5-1.5B' = 35.3 (divided by 100). BBH 3-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "hellaswag",
    "score": 0.661,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HellaSwag', column 'SmolLM2-1.7B' = 66.1 (divided by 100). HellaSwag, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "hellaswag",
    "score": 0.561,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HellaSwag', column 'Llama3.2-1B' = 56.1 (divided by 100). HellaSwag, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "hellaswag",
    "score": 0.609,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HellaSwag', column 'Qwen2.5-1.5B' = 60.9 (divided by 100). HellaSwag, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "piqa",
    "score": 0.744,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'PIQA', column 'SmolLM2-1.7B' = 74.4 (divided by 100). PIQA, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "piqa",
    "score": 0.723,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'PIQA', column 'Llama3.2-1B' = 72.3 (divided by 100). PIQA, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "piqa",
    "score": 0.732,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'PIQA', column 'Qwen2.5-1.5B' = 73.2 (divided by 100). PIQA, zero-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "gsm8k",
    "score": 0.488,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'GSM8K (5-shot)', column 'SmolLM2-1.7B' = 48.8 (divided by 100). GSM8K 5-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gsm8k",
    "score": 0.374,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'GSM8K (5-shot)', column 'Llama3.2-1B' = 37.4 (divided by 100). GSM8K 5-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "gsm8k",
    "score": 0.633,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'GSM8K (5-shot)', column 'Qwen2.5-1.5B' = 63.3 (divided by 100). GSM8K 5-shot. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "math-subset-unspecified",
    "score": 0.21,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'MATH (4-shot)', column 'SmolLM2-1.7B' = 21.0 (divided by 100). MATH 4-shot (subset not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "math-subset-unspecified",
    "score": 0.195,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'MATH (4-shot)', column 'Llama3.2-1B' = 19.5 (divided by 100). MATH 4-shot (subset not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "math-subset-unspecified",
    "score": 0.196,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'MATH (4-shot)', column 'Qwen2.5-1.5B' = 19.6 (divided by 100). MATH 4-shot (subset not stated). Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "smollm2-1-7b",
    "bench": "humaneval",
    "score": 0.281,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "lab",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HumanEval', column 'SmolLM2-1.7B' = 28.1 (divided by 100). HumanEval. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "humaneval",
    "score": 0.335,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HumanEval', column 'Llama3.2-1B' = 33.5 (divided by 100). HumanEval. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "humaneval",
    "score": 0.305,
    "source": "https://arxiv.org/html/2502.02737v1",
    "by": "independent",
    "date": "2025-02-04",
    "note": "SmolLM2 paper (Ben Allal et al., arXiv 2502.02737v1), Table 5 'Comparison of 1-2B instruction-tuned models'; row 'HumanEval', column 'Qwen2.5-1.5B' = 30.5 (divided by 100). HumanEval. Instruct checkpoints: §5.4 'We evaluate the final instruct version of SmolLM2 and compare it with the instruct variants of Qwen2.5-1.5B and Llama3.2-1B'; the paper's evaluations use lighteval. MT-Bench, OpenRewrite-Eval, the ARC row (easy/challenge aggregation not stated) and the MMLU-Pro row (lighteval MCF variant; its two independent rows would otherwise become the catalogue's canonical 'mmlu-pro' series) not imported.",
    "source_id": "paper-2502.02737v1"
  },
  {
    "model": "gemini-2-0-pro-exp-02-05",
    "bench": "aider-polyglot",
    "score": 0.356,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-02-25",
    "note": "Published entry 'Gemini 2.0 Pro exp-02-05'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.75.2.dev; evaluator commit=2fccd47; command=aider --model gemini/gemini-2.0-pro-exp-02-05; run=2025-02-25-20-23-07--gemini-pro. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "aider-polyglot",
    "score": 0.036,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-21",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=a755079-dirty; command=aider --model gpt-4o-mini-2024-07-18; run=2024-12-21-18-41-18--polyglot-gpt-4o-mini. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "aider-polyglot",
    "score": 0.516,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-01-17",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.71.2.dev; evaluator commit=6451d59; command=aider --model claude-3-5-sonnet-20241022; run=2025-01-17-19-44-33--sonnet-baseline-jan-17. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "aider-polyglot",
    "score": 0.182,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-30",
    "note": "Published entry 'gpt-4o-2024-11-20'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.70.1.dev; evaluator commit=09ee197-dirty; command=aider --model gpt-4o-2024-11-20; run=2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "aider-polyglot",
    "score": 0.231,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-30",
    "note": "Published entry 'gpt-4o-2024-08-06'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.70.1.dev; evaluator commit=09ee197-dirty; command=aider --model gpt-4o-2024-08-06; run=2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "aider-polyglot",
    "score": 0.617,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-21",
    "note": "Published entry 'o1-2024-12-17 (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.69.2.dev; evaluator commit=a755079-dirty; command=aider --model openrouter/openai/o1; run=2024-12-21-19-23-03--polyglot-o1-hard-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. test_cases=224 of 225 completed (published pass_rate_2 as reported).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "aider-polyglot",
    "score": 0.28,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-21",
    "note": "Published entry 'claude-3-5-haiku-20241022'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.69.2.dev; evaluator commit=a755079-dirty; command=aider --model claude-3-5-haiku-20241022; run=2024-12-21-21-46-27--polyglot-haiku-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o1-mini",
    "bench": "aider-polyglot",
    "score": 0.329,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-22",
    "note": "Published entry 'o1-mini-2024-09-12'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=37df899; command=aider --model o1-mini; run=2024-12-22-21-26-35--polyglot-o1mini-whole. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "yi-lightning",
    "bench": "aider-polyglot",
    "score": 0.129,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-23",
    "note": "Published entry 'yi-lightning'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=2b1625e; command=aider --model openai/yi-lightning; run=2024-12-23-01-11-56--yi-test. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "deepseek-v3",
    "bench": "aider-polyglot",
    "score": 0.484,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-25",
    "note": "Published entry 'DeepSeek Chat V3 (prev)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.69.2.dev; evaluator commit=0a23c4a-dirty; command=aider --model deepseek/deepseek-chat; run=2024-12-25-13-31-51--deepseekv3preview-diff2. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Run dated 2024-12-25 and labelled V3 '(prev)' to distinguish it from the file's later 'DeepSeek V3 (0324)' entry: the original V3, before the 0324 update; endpoint is not a weight digest.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "aider-polyglot",
    "score": 0.218,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-01-28",
    "note": "Published entry 'qwen-max-2025-01-25'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.72.4.dev; evaluator commit=ae7d459; command=OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25; run=2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o3-mini",
    "bench": "aider-polyglot",
    "score": 0.604,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Published entry 'o3-mini (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.72.4.dev; evaluator commit=b0d58d1-dirty; command=aider --model o3-mini --reasoning-effort high; run=2025-01-31-20-42-47--o3-mini-diff-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. test_cases=224 of 225 completed (published pass_rate_2 as reported).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-0-flash-thinking-exp-01-21",
    "bench": "aider-polyglot",
    "score": 0.182,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-01-21",
    "note": "Published entry 'gemini-2.0-flash-thinking-exp-01-21'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.72.2.dev; evaluator commit=843720a; command=aider --model gemini/gemini-2.0-flash-thinking-exp-01-21; run=2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "aider-polyglot",
    "score": 0.649,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-02-24",
    "note": "Published entry 'claude-3-7-sonnet-20250219 (32k thinking tokens)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.75.1.dev; evaluator commit=60d11a6, 93edbda; command=aider --model anthropic/claude-3-7-sonnet-20250219 --thinking-tokens 32k; run=2025-02-24-21-47-23--sonnet37-diff-think-32k-64k. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "aider-polyglot",
    "score": 0.449,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-02-27",
    "note": "Published entry 'gpt-4.5-preview'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.75.2.dev; evaluator commit=b462e55-dirty; command=aider --model openai/gpt-4.5-preview; run=2025-02-27-20-26-15--gpt45-diff3. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. test_cases=224 of 225 completed (published pass_rate_2 as reported).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "qwq-32b",
    "bench": "aider-polyglot",
    "score": 0.209,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-03-06",
    "note": "Published entry 'QwQ-32B'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.75.3.dev; evaluator commit=51d118f-dirty; command=aider --model fireworks_ai/accounts/fireworks/models/qwq-32b; run=2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "grok-3-beta",
    "bench": "aider-polyglot",
    "score": 0.533,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Published entry 'Grok 3 Beta'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.81.2.dev; evaluator commit=2dd40fc-dirty; command=aider --model openrouter/x-ai/grok-3-beta; run=2025-04-10-04-21-31--grok3-diff-exuser. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "aider-polyglot",
    "score": 0.493,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-10",
    "note": "Published entry 'Grok 3 Mini Beta (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.81.3.dev; evaluator commit=8ee33da-dirty; command=aider --model xai/grok-3-mini-beta --reasoning-effort high; run=2025-04-10-23-59-02--xai-grok3-mini-whole-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "aider-polyglot",
    "score": 0.089,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'gpt-4.1-nano'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.81.4.dev; evaluator commit=71d1591-dirty; command=aider --model gpt-4.1-nano; run=2025-04-14-22-46-01--gpt41nano-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "aider-polyglot",
    "score": 0.471,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-20",
    "note": "Published entry 'gemini-2.5-flash-preview-04-17 (default)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.82.3.dev; evaluator commit=7fcce5d-dirty; command=aider --model gemini/gemini-2.5-flash-preview-04-17; run=2025-04-20-19-54-31--flash25-diff-no-think. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "aider-polyglot",
    "score": 0.769,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-05-07",
    "note": "Published entry 'Gemini 2.5 Pro Preview 05-06'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff-fenced; Aider version=0.82.4.dev; evaluator commit=3b08327-dirty; command=aider --model gemini/gemini-2.5-pro-preview-05-06; run=2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "aider-polyglot",
    "score": 0.613,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-05-24",
    "note": "Published entry 'claude-sonnet-4-20250514 (32k thinking)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.83.3.dev; evaluator commit=e3cb907; command=aider --model claude-sonnet-4-20250514; run=2025-05-24-22-10-36--sonnet4-diff-exuser-think32k. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "aider-polyglot",
    "score": 0.72,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-05-25",
    "note": "Published entry 'claude-opus-4-20250514 (32k thinking)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.83.3.dev; evaluator commit=9ef3211; command=aider --model claude-opus-4-20250514; run=2025-05-25-20-40-51--opus4-diff-exuser. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "aider-polyglot",
    "score": 0.551,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-05-25",
    "note": "Published entry 'gemini-2.5-flash-preview-05-20 (24k think)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.83.3.dev; evaluator commit=a8568c3-dirty; command=aider --model gemini/gemini-2.5-flash-preview-05-20; run=2025-05-25-22-58-44--flash25-05-20-24k-think. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "aider-polyglot",
    "score": 0.831,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-06-06",
    "note": "Published entry 'gemini-2.5-pro-preview-06-05 (32k think)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff-fenced; Aider version=0.84.1.dev; evaluator commit=f827f22; command=aider --model gemini/gemini-2.5-pro-preview-06-05 --thinking-tokens 32k; run=2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aider-polyglot",
    "score": 0.714,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-06-06",
    "note": "Published entry 'DeepSeek R1 (0528)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.84.1.dev; evaluator commit=4c161f9-dirty; command=aider --model deepseek/deepseek-reasoner; run=2025-06-06-16-47-07--r1-diff. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. test_cases=224 of 225 completed (published pass_rate_2 as reported).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "o3-pro",
    "bench": "aider-polyglot",
    "score": 0.849,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-06-28",
    "note": "Published entry 'o3-pro (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.85.1.dev; evaluator commit=5318380; command=aider --model o3-pro; run=2025-06-28-00-38-18--o3-pro-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "grok-4-0709",
    "bench": "aider-polyglot",
    "score": 0.796,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-07-11",
    "note": "Published entry 'grok-4 (high)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.85.2.dev; evaluator commit=f7870b6-dirty; command=aider --model openrouter/x-ai/grok-4; run=2025-07-11-19-37-40--xai-or-grok4-high. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "aider-polyglot",
    "score": 0.742,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-10-03",
    "note": "Published entry 'DeepSeek-V3.2-Exp (Reasoner)'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff; Aider version=0.86.2.dev; evaluator commit=cbb5376; command=aider --model deepseek/deepseek-reasoner; run=2025-10-03-09-45-34--deepseek-v3.2-reasoner. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.8271,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Grok 4.1 Fast Reasoning'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau2-verified-gpt51-user",
    "score": 0.7939,
    "source": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Amazon AGI τ²-Bench-Verified README leaderboard; exact row 'Gemini Pro 3'; published aggregate Average across airline/retail/telecom. Corrected dataset, gpt-5.1 user simulator for all models. Highest explicitly listed reasoning effort selected before reading outcomes. Date is retrieval, individual run date not stated. No component rows duplicated. Gemini 3 Pro was released only as the gemini-3-pro-preview API model; not Gemini 3.1 Pro.",
    "source_id": "amazon-tau2-verified"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.07125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-3-haiku",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-3-haiku' (Anthropic via 'anthropic/claude-3-haiku'; only one Claude 3 Haiku snapshot (20240307) exists; API model in attempt metadata: anthropic/claude-3-haiku). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 28.5 / 400); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-17. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-3-haiku",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-3-haiku' (Anthropic via 'anthropic/claude-3-haiku'; only one Claude 3 Haiku snapshot (20240307) exists; API model in attempt metadata: anthropic/claude-3-haiku). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 0 / 120); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-17. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.6625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-4-mini-high",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-4-mini-high' (reasoning effort high; the xhigh folder is incomplete (389/400); API model in attempt metadata: gpt-5.4-mini). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 265 / 400); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-18. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.008333,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-4-mini-low",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-4-mini-low' (reasoning effort low; the xhigh (93/120), high (116/120) and medium (117/120) folders are incomplete; API model in attempt metadata: gpt-5.4-mini). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 1 / 120); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-18. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.51625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-4-nano-high",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-4-nano-high' (reasoning effort high; the xhigh folder is incomplete (260/400); API model in attempt metadata: gpt-5.4-nano). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 206.5 / 400); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-18. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.034722,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-4-nano-medium",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-4-nano-medium' (reasoning effort medium; the xhigh (20/120) and high (113/120) folders are incomplete; API model in attempt metadata: gpt-5.4-nano). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 4.1667 / 120); method reproduces the published results.json exactly for every folder that has one. Attempts dated 2026-03-18. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-2-pro",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.94625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-2-pro-2025-12-11-high/results.json",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-2-pro-2025-12-11-high' (reasoning effort high; the xhigh folder is incomplete (355/400); API model in attempt metadata: gpt-5.2-pro-2025-12-11). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). published results.json: score 378.5 over total_tasks 400 (sum of per-task scores), divided by 400. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-2-pro",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.379167,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-2-pro-2025-12-11-medium/results.json",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-2-pro-2025-12-11-medium' (reasoning effort medium; the high folder is incomplete (119/120) and no xhigh folder exists; API model in attempt metadata: gpt-5.2-pro-2025-12-11). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). published results.json: score 45.5 over total_tasks 120 (sum of per-task scores), divided by 120. Highest-effort complete run for this model in this dataset; incomplete higher-effort runs are not imported. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "o3-mini",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.661,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'o3-mini-2025-01-31-high' = 66.1% (CI --1.5/++2.1); win rate vs baseline o3-mini-2025-01-31 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 66.1,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.61,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'o1-2024-12-17-high' = 61.0% (CI --2.0/++2.1); win rate vs baseline o3-mini-2025-01-31 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 61,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.598,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'claude-3-7-sonnet-20250219-thinking-16k' = 59.8% (CI --2.0/++1.8); win rate vs baseline o3-mini-2025-01-31 (50%); extended thinking, 16k budget (only listed setting). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 59.8,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.5,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'gpt-4.5-preview' = 50.0% (CI --1.9/++2.0); win rate vs baseline o3-mini-2025-01-31 (50%); unversioned alias, only one GPT-4.5 Preview snapshot (2025-02-27) existed. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 50,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwq-32b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.435,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'QwQ-32B' = 43.5% (CI --2.5/++2.1); win rate vs baseline o3-mini-2025-01-31 (50%); final March 2025 QwQ-32B, not the Preview. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 43.5,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.339,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Qwen3-30B-A3B' = 33.9% (CI --1.6/++1.5); win rate vs baseline o3-mini-2025-01-31 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 33.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.33,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'claude-3-5-sonnet-20241022' = 33.0% (CI --2.3/++1.8); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 33,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "athene-v2-chat",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.164,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Athene-V2-Chat' = 16.4% (CI --1.4/++1.4); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 16.4,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwen3-4b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.15,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Qwen3-4B' = 15.0% (CI --1.1/++1.5); win rate vs baseline o3-mini-2025-01-31 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 15,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.137,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'gpt-4.1-nano' = 13.7% (CI --1.1/++1.0); win rate vs baseline o3-mini-2025-01-31 (50%); unversioned alias, OpenAI has published only one GPT-4.1 nano snapshot (2025-04-14). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 13.7,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.103,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'Llama-3.1-Nemotron-70B-Instruct-HF' = 10.3% (CI --0.8/++1.0); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 10.3,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.639,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'claude-3-7-sonnet-20250219-thinking-16k' = 63.9% (CI --1.7/++1.9); win rate vs baseline gemini-2.0-flash-001 (50%); extended thinking, 16k budget (only listed setting). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 63.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwq-32b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.609,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'QwQ-32B' = 60.9% (CI --2.0/++1.6); win rate vs baseline gemini-2.0-flash-001 (50%); final March 2025 QwQ-32B, not the Preview. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 60.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.599,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'o1-2024-12-17-high' = 59.9% (CI --2.1/++1.7); win rate vs baseline gemini-2.0-flash-001 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 59.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.514,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'gpt-4.5-preview' = 51.4% (CI --1.9/++2.0); win rate vs baseline gemini-2.0-flash-001 (50%); unversioned alias, only one GPT-4.5 Preview snapshot (2025-02-27) existed. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 51.4,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "o3-mini",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.43,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'o3-mini-2025-01-31-high' = 43.0% (CI --1.7/++2.1); win rate vs baseline gemini-2.0-flash-001 (50%); reasoning effort high (highest listed). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 43,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.349,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Qwen3-30B-A3B' = 34.9% (CI --2.0/++1.6); win rate vs baseline gemini-2.0-flash-001 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 34.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.269,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Llama-3.1-Nemotron-70B-Instruct-HF' = 26.9% (CI --2.0/++1.8); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 26.9,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.242,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'claude-3-5-sonnet-20241022' = 24.2% (CI --1.5/++1.5); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 24.2,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "athene-v2-chat",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.181,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Athene-V2-Chat' = 18.1% (CI --1.6/++1.5); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 18.1,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "qwen3-4b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.132,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'Qwen3-4B' = 13.2% (CI --1.2/++1.2); win rate vs baseline gemini-2.0-flash-001 (50%); April 2025 hybrid release; thinking mode not stated. Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 13.2,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.107,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'gpt-4.1-nano' = 10.7% (CI --1.1/++1.1); win rate vs baseline gemini-2.0-flash-001 (50%); unversioned alias, OpenAI has published only one GPT-4.1 nano snapshot (2025-04-14). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01.",
    "raw_value": 10.7,
    "scale_divisor": 100,
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "bfcl-v4",
    "score": 0.6957,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Grok-4-1-fast-reasoning (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: grok-4-1-fast-reasoning.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "bfcl-v4",
    "score": 0.6814,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Gemini-3-Pro-Preview (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gemini-3-pro-preview.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "grok-4-0709",
    "bench": "bfcl-v4",
    "score": 0.6138,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Grok-4-0709 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: grok-4-0709.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "bfcl-v4",
    "score": 0.3756,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Mistral-Medium-2505 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: mistral-medium-2505.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "bfcl-v4",
    "score": 0.3305,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'GPT-4.1-nano-2025-04-14 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: gpt-4.1-nano-2025-04-14.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "bfcl-v4",
    "score": 0.3207,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Command R7B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: command-r7b-12-2024.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-pro",
    "bench": "bfcl-v4",
    "score": 0.2497,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Amazon-Nova-Pro-v1:0 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: us.amazon.nova-pro-v1:0.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3174,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'QwQ-32B-Preview' (model link https://huggingface.co/Qwen/QwQ-32B-Preview); 'Average Performance' 31.74 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "qwq-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3937,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'QWQ-32B' (model link https://huggingface.co/Qwen/QwQ-32B); 'Average Performance' 39.37 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "qwen3-4b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.385,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-4B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-4B); 'Average Performance' 38.5 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4093,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'Qwen3-30B-A3B-Thinking' (model link https://huggingface.co/Qwen/Qwen3-30B-A3B); 'Average Performance' 40.93 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Hybrid Qwen3 checkpoint evaluated with enable_thinking=True (the -Thinking row, highest reasoning setting; the -Non-Thinking row is not imported). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.353,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gpt-35-turbo-0125' (model link https://platform.openai.com/docs/models/gpt-3.5-turbo); 'Average Performance' 35.3 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Label uses Azure OpenAI naming ('gpt-35-turbo', version 0125) for the dated snapshot gpt-3.5-turbo-0125. Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.442,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gpt-4o-0806' (model link https://platform.openai.com/docs/models/gpt-4o); 'Average Performance' 44.2 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Label 'gpt-4o-0806' is the dated snapshot gpt-4o-2024-08-06 (models-log release date 2024-08-06). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4303,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemini-2.0-flash-001' (model link https://deepmind.google/technologies/gemini/); 'Average Performance' 43.03 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4484,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemini-2.5-flash' (model link https://deepmind.google/technologies/gemini/); 'Average Performance' 44.84 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). 'gemini-2.5-flash' is the stable (GA) model code; preview snapshots carry -preview-MM-DD suffixes. Added in the 2025-07-22 update, after GA. Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4385,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'gemini-1.5-pro-002' (model link https://deepmind.google/technologies/gemini/); 'Average Performance' 43.85 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "eqbench3-elo",
    "score": 1304.66,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gemini-2.5-pro-preview-06-05'; raw TrueSkill rating 'elo' 1304.66 (sigma 4.91, 95% CI 1295.03-1314.29); normalised display value elo_norm 1549.14 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact dated preview API id.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "eqbench3-elo",
    "score": 1082.9,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'Qwen/Qwen3-30B-A3B'; raw TrueSkill rating 'elo' 1082.9 (sigma 4.18, 95% CI 1074.71-1091.09); normalised display value elo_norm 880.96 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Original April 2025 hybrid checkpoint Qwen/Qwen3-30B-A3B (not the 2507 Instruct/Thinking releases).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "eqbench3-elo",
    "score": 1169.2,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'anthropic/claude-3.7-sonnet'; raw TrueSkill rating 'elo' 1169.2 (sigma 4.13, 95% CI 1161.11-1177.29); normalised display value elo_norm 1140.98 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned alias; Anthropic released only one Claude 3.7 Sonnet snapshot (claude-3-7-sonnet-20250219).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "eqbench3-elo",
    "score": 1260.39,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'anthropic/claude-opus-4'; raw TrueSkill rating 'elo' 1260.39 (sigma 4.48, 95% CI 1251.61-1269.18); normalised display value elo_norm 1415.76 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned alias; Anthropic released only one Claude Opus 4 snapshot (claude-opus-4-20250514).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "eqbench3-elo",
    "score": 1224.26,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'claude-sonnet-4'; raw TrueSkill rating 'elo' 1224.26 (sigma 4.65, 95% CI 1215.14-1233.37); normalised display value elo_norm 1306.87 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned alias; Anthropic released only one Claude Sonnet 4 snapshot (claude-sonnet-4-20250514).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "eqbench3-elo",
    "score": 1096.59,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemini-2.0-flash-001'; raw TrueSkill rating 'elo' 1096.59 (sigma 3.28, 95% CI 1090.16-1103.01); normalised display value elo_norm 922.19 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact versioned API id gemini-2.0-flash-001.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "eqbench3-elo",
    "score": 975.33,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'google/gemma-2-9b-it'; raw TrueSkill rating 'elo' 975.33 (sigma 6.71, 95% CI 962.18-988.47); normalised display value elo_norm 556.81 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact open-weights checkpoint google/gemma-2-9b-it.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "gpt-4-0314",
    "bench": "eqbench3-elo",
    "score": 984.73,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gpt-4-0314'; raw TrueSkill rating 'elo' 984.73 (sigma 6.03, 95% CI 972.92-996.54); normalised display value elo_norm 585.15 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact dated API snapshot.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "eqbench3-elo",
    "score": 1134.94,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gpt-4.1-nano'; raw TrueSkill rating 'elo' 1134.94 (sigma 3.97, 95% CI 1127.16-1142.71); normalised display value elo_norm 1037.74 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned alias; OpenAI has published only one GPT-4.1 nano snapshot (gpt-4.1-nano-2025-04-14).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "eqbench3-elo",
    "score": 1153.73,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'gpt-4.5-preview-2025-02-27'; raw TrueSkill rating 'elo' 1153.73 (sigma 4.62, 95% CI 1144.69-1162.78); normalised display value elo_norm 1094.38 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact dated API snapshot.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "eqbench3-elo",
    "score": 1146.85,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'grok-3-mini-beta'; raw TrueSkill rating 'elo' 1146.85 (sigma 4.43, 95% CI 1138.17-1155.54); normalised display value elo_norm 1073.64 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact API id grok-3-mini-beta.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "grok-4-0709",
    "bench": "eqbench3-elo",
    "score": 1190.35,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'grok-4'; raw TrueSkill rating 'elo' 1190.35 (sigma 5.59, 95% CI 1179.39-1201.31); normalised display value elo_norm 1204.71 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned alias; xAI released one Grok 4 snapshot (grok-4-0709) before Grok 4 Fast/4.1.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "mistral-small-3",
    "bench": "eqbench3-elo",
    "score": 1058.88,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'mistralai/mistral-small-24b-instruct-2501'; raw TrueSkill rating 'elo' 1058.88 (sigma 4.21, 95% CI 1050.63-1067.14); normalised display value elo_norm 808.58 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact checkpoint Mistral-Small-24B-Instruct-2501.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "eqbench3-elo",
    "score": 1055.92,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'mistralai/mistral-small-3.1-24b-instruct'; raw TrueSkill rating 'elo' 1055.92 (sigma 4.65, 95% CI 1046.8-1065.03); normalised display value elo_norm 799.64 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Unversioned slug; Mistral released one Small 3.1 checkpoint (Mistral-Small-3.1-24B-Instruct-2503).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "qwq-32b",
    "bench": "eqbench3-elo",
    "score": 1197.52,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'qwen/qwq-32b'; raw TrueSkill rating 'elo' 1197.52 (sigma 3.51, 95% CI 1190.64-1204.39); normalised display value elo_norm 1226.3 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Final QwQ-32B (March 2025), not QwQ-32B-Preview.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "grok-3-beta",
    "bench": "eqbench3-elo",
    "score": 1176.62,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'x-ai/grok-3-beta'; raw TrueSkill rating 'elo' 1176.62 (sigma 3.85, 95% CI 1169.08-1184.16); normalised display value elo_norm 1163.34 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Exact API id grok-3-beta.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "eqbench3-elo",
    "score": 1287.22,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'deepseek/deepseek-v4-flash'; raw TrueSkill rating 'elo' 1287.22 (sigma 5.13, 95% CI 1277.15-1297.28); normalised display value elo_norm 1496.58 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. API id deepseek-v4-flash; the data file's last_updated (2026-05-10) predates 2026-07-31, when DeepSeek-V4-Flash-0731 replaced the April 2026 preview under that id, so this is the preview checkpoint.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "eqbench3-elo",
    "score": 1329.68,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'deepseek/deepseek-v4-pro'; raw TrueSkill rating 'elo' 1329.68 (sigma 5.25, 95% CI 1319.39-1339.97); normalised display value elo_norm 1624.52 not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. API id deepseek-v4-pro; the data file's last_updated (2026-05-10) predates 2026-08-13, when the 0813 GA replaced the April 2026 preview under that id, so this is the preview checkpoint.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7880508474576271,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/01-ai--Yi-Coder-9B-Chat_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry '01-ai--Yi-Coder-9B-Chat'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7411016949152542,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--CodeQwen1.5-7B-Chat_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--CodeQwen1.5-7B-Chat'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-27",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8372033898305085,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--QwQ-32B-Preview_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'Qwen--QwQ-32B-Preview'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-11-30",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8800847457627118,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/claude-3-5-sonnet-20240620_bedrock_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=bedrock; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-11-04",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7904237288135593,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/deepseek-ai--DeepSeek-Coder-V2-Lite-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'deepseek-ai--DeepSeek-Coder-V2-Lite-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7750847457627118,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/deepseek-ai--deepseek-coder-33b-instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'deepseek-ai--deepseek-coder-33b-instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7357627118644068,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/deepseek-ai--deepseek-coder-6.7b-instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'deepseek-ai--deepseek-coder-6.7b-instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7763559322033899,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/gemini-1.5-flash-002_google_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-002'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=google; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8371186440677967,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/gemini-1.5-pro-002_google_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-002'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=google; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2-27b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8023728813559322,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/google--gemma-2-27b-it_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'google--gemma-2-27b-it'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2-9b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6807627118644067,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/google--gemma-2-9b-it_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'google--gemma-2-9b-it'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8665254237288136,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/gpt-4o-2024-08-06_openai_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=openai; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8551694915254237,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/gpt-4o-mini-2024-07-18_openai_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=openai; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.4369491525423729,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/meta-llama--Meta-Llama-3-8B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'meta-llama--Meta-Llama-3-8B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-27",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6841525423728814,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/microsoft--Phi-3-medium-4k-instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'microsoft--Phi-3-medium-4k-instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-11-03",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "evalperf-correctness-20240328",
    "score": 0.7283898305084746,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/mistralai--Codestral-22B-v0.1_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'mistralai--Codestral-22B-v0.1'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-large-2407",
    "bench": "evalperf-correctness-20240328",
    "score": 0.824322033898305,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/mistralai--Mistral-Large-Instruct-2407_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'mistralai--Mistral-Large-Instruct-2407'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "evalperf-correctness-20240328",
    "score": 0.5738983050847457,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/mistralai--Mistral-Nemo-Instruct-2407_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'mistralai--Mistral-Nemo-Instruct-2407'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-small-2409",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6951694915254236,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/mistralai--Mistral-Small-Instruct-2409_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'mistralai--Mistral-Small-Instruct-2409'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-19",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6183050847457627,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/nvidia--Llama-3.1-Nemotron-70B-Instruct-HF_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'nvidia--Llama-3.1-Nemotron-70B-Instruct-HF'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export.",
    "date": "2024-10-24",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-mini",
    "bench": "evalperf-correctness-20240328",
    "score": 0.8883050847457628,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/o1-mini-2024-09-12_openai_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 50 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=openai; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export. Note: this run uses 50 samples/task.",
    "date": "2024-11-04",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.953,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'claude-3-7-sonnet-20250219'; column 'Model Avg' = 0.953; single-turn 0.96, multi-turn 0.95; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.938,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.0-flash-001'; column 'Model Avg' = 0.938; single-turn 0.95, multi-turn 0.93; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.9,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4o-2024-11-20'; column 'Model Avg' = 0.900; single-turn 0.92, multi-turn 0.88; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.9,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.5-preview-2025-02-27'; column 'Model Avg' = 0.900; single-turn 0.93, multi-turn 0.87; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.876,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'o1-2024-12-17'; column 'Model Avg' = 0.876; single-turn 0.83, multi-turn 0.92; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "nova-pro",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.868,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'amazon.nova-pro-v1'; column 'Model Avg' = 0.868; single-turn 0.94, multi-turn 0.79; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Amazon Bedrock model id amazon.nova-pro-v1.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "nova-lite",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.868,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'amazon.nova-lite-v1'; column 'Model Avg' = 0.868; single-turn 0.91, multi-turn 0.83; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Amazon Bedrock model id amazon.nova-lite-v1.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.847,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'o3-mini-2025-01-31'; column 'Model Avg' = 0.847; single-turn 0.80, multi-turn 0.90; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Reasoning effort not stated (API default medium).",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.832,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4o-mini'; column 'Model Avg' = 0.832; single-turn 0.85, multi-turn 0.82; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18).",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.803,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-nano-2025-04-14'; column 'Model Avg' = 0.803; single-turn 0.85, multi-turn 0.76; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.801,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'claude-3-5-sonnet-20241022'; column 'Model Avg' = 0.801; single-turn 0.83, multi-turn 0.77; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.765,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'claude-3-5-haiku-20241022'; column 'Model Avg' = 0.765; single-turn 0.78, multi-turn 0.75; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "mistral-medium-3-1",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.61,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'mistral-medium-2508'; column 'Avg AC' = 0.61; Avg TSQ 0.77; per-domain AC banking 0.57, healthcare 0.6, insurance 0.7, investment 0.57, telecom 0.59; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.55,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'claude-sonnet-4-20250514'; column 'Avg AC' = 0.55; Avg TSQ 0.92; per-domain AC banking 0.58, healthcare 0.62, insurance 0.53, investment 0.49, telecom 0.53; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "grok-4-0709",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.42,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'grok-4-0709'; column 'Avg AC' = 0.42; Avg TSQ 0.88; per-domain AC banking 0.29, healthcare 0.4, insurance 0.48, investment 0.5, telecom 0.42; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.38,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gpt-4.1-nano-2025-04-14'; column 'Avg AC' = 0.38; Avg TSQ 0.63; per-domain AC banking 0.4, healthcare 0.4, insurance 0.41, investment 0.29, telecom 0.38; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "magistral-medium-2506",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.32,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'magistral-medium-2506'; column 'Avg AC' = 0.32; Avg TSQ 0.59; per-domain AC banking 0.3, healthcare 0.35, insurance 0.38, investment 0.26, telecom 0.3; Output Type 'Reasoning'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "nova-pro",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.29,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'nova-pro-v1'; column 'Avg AC' = 0.29; Avg TSQ 0.65; per-domain AC banking 0.33, healthcare 0.29, insurance 0.39, investment 0.17, telecom 0.29; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "nova-lite",
    "bench": "galileo-agent-v2-action-completion",
    "score": 0.16,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv",
    "by": "independent",
    "date": "2025-11-18",
    "note": "Galileo Agent Leaderboard Space (results_v2.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'nova-lite-v1'; column 'Avg AC' = 0.16; Avg TSQ 0.55; per-domain AC banking 0.12, healthcare 0.18, insurance 0.19, investment 0.15, telecom 0.18; Output Type 'Normal'; simulated multi-turn enterprise agent scenarios, Action Completion judged by Galileo's LLM metric.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "aragen-v3-3c3h",
    "score": 0.2442,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.2-11B-Vision-Instruct'; 3C3H Score 0.2442 (95% CI 0.206704-0.282962, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_074429; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "aragen-v3-3c3h",
    "score": 0.5051,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'meta-llama/Llama-3.2-90B-Vision-Instruct'; 3C3H Score 0.5051 (95% CI 0.454901-0.55482, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_114723; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "aragen-v3-3c3h",
    "score": 0.7558,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-sonnet-4-20250514'; 3C3H Score 0.7558 (95% CI 0.720133-0.789533, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251128_085938; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "aragen-v3-3c3h",
    "score": 0.7816,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-3-7-sonnet-20250219'; 3C3H Score 0.7816 (95% CI 0.747146-0.814022, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251128_200834; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "aragen-v3-3c3h",
    "score": 0.8096,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-28",
    "note": "Inception AraGen-v3 leaderboard entry 'claude-opus-4-20250514'; 3C3H Score 0.8096 (95% CI 0.781344-0.836776, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251128_232155; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "aragen-v3-3c3h",
    "score": 0.8429,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-29",
    "note": "Inception AraGen-v3 leaderboard entry 'o1-2024-12-17'; 3C3H Score 0.8429 (95% CI 0.812004-0.871673, n=321); judge claude-sonnet-4-20250514; 321/324 entries answered; run 20251129_091723; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o1-mini",
    "bench": "aragen-v3-3c3h",
    "score": 0.7046,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-29",
    "note": "Inception AraGen-v3 leaderboard entry 'o1-mini-2024-09-12'; 3C3H Score 0.7046 (95% CI 0.662543-0.746004, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251129_232621; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o3-mini",
    "bench": "aragen-v3-3c3h",
    "score": 0.5981,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Inception AraGen-v3 leaderboard entry 'o3-mini-2025-01-31'; 3C3H Score 0.5981 (95% CI 0.551258-0.643755, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20251130_071914; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "aragen-v3-3c3h",
    "score": 0.5074,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Inception AraGen-v3 leaderboard entry 'gpt-4o-mini-2024-07-18'; 3C3H Score 0.5074 (95% CI 0.45743-0.555301, n=321); judge claude-sonnet-4-20250514; 321/324 entries answered; run 20251130_203645; RM_Thinking_Traces=None.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "aragen-v3-3c3h",
    "score": 0.6415,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-11-30",
    "note": "Inception AraGen-v3 leaderboard entry 'gemini-3-pro-preview'; 3C3H Score 0.6415 (95% CI 0.603444-0.679883, n=306); judge claude-sonnet-4-20250514; 306/324 entries answered; run 20251130_233940; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "aragen-v3-3c3h",
    "score": 0.6837,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-12-01",
    "note": "Inception AraGen-v3 leaderboard entry 'gemini-2.5-pro-preview-05-06'; 3C3H Score 0.6837 (95% CI 0.644847-0.721368, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251201_043335; RM_Thinking_Traces=None. Reasoning effort not stated in the results file (API default assumed unknown).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7077,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-3-7-sonnet-20250219'; 3C3H Score 0.7077 (95% CI 0.663881-0.749653, n=268); judge claude-3.5-sonnet; 268/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5689,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4.1-nano-2025-04-14'; 3C3H Score 0.5689 (95% CI 0.518747-0.618605, n=263); judge claude-3.5-sonnet; 263/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7445,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4o-2024-08-06'; 3C3H Score 0.7445 (95% CI 0.699485-0.788014, n=262); judge claude-3.5-sonnet; 262/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7365,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gemini-2.5-flash-preview-05-20'; 3C3H Score 0.7365 (95% CI 0.694503-0.777685, n=265); judge claude-3.5-sonnet; 265/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7244,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4o-2024-11-20'; 3C3H Score 0.7244 (95% CI 0.682602-0.76432, n=266); judge claude-3.5-sonnet; 266/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "hindigen-v1-3c3h",
    "score": 0.655,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4o-mini-2024-07-18'; 3C3H Score 0.655 (95% CI 0.606344-0.700863, n=262); judge claude-3.5-sonnet; 262/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o1-mini",
    "bench": "hindigen-v1-3c3h",
    "score": 0.6,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'o1-mini-2024-09-12'; 3C3H Score 0.6 (95% CI 0.553865-0.646021, n=263); judge claude-3.5-sonnet; 263/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "hindigen-v1-3c3h",
    "score": 0.6975,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-sonnet-4-20250514'; 3C3H Score 0.6975 (95% CI 0.655227-0.737785, n=267); judge claude-3.5-sonnet; 267/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "mistral-small-3",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5029,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'mistralai/Mistral-Small-24B-Instruct-2501'; 3C3H Score 0.5029 (95% CI 0.450311-0.554819, n=263); judge claude-3.5-sonnet; 263/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7177,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gemini-2.5-pro-preview-05-06'; 3C3H Score 0.7177 (95% CI 0.678391-0.75512, n=269); judge claude-3.5-sonnet; 269/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7964,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'o1-2024-12-17'; 3C3H Score 0.7964 (95% CI 0.755761-0.835532, n=265); judge claude-3.5-sonnet; 265/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5532,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-3-haiku-20240307'; 3C3H Score 0.5532 (95% CI 0.504511-0.600358, n=262); judge claude-3.5-sonnet; 262/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "o3-mini",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5514,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'o3-mini-2025-01-31'; 3C3H Score 0.5514 (95% CI 0.497881-0.604065, n=262); judge claude-3.5-sonnet; 262/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "hindigen-v1-3c3h",
    "score": 0.491,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-3.5-turbo-0125'; 3C3H Score 0.491 (95% CI 0.438954-0.54236, n=263); judge claude-3.5-sonnet; 263/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "hindigen-v1-3c3h",
    "score": 0.6125,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-3-5-haiku-20241022'; 3C3H Score 0.6125 (95% CI 0.564403-0.659199, n=262); judge claude-3.5-sonnet; 262/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7398,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'gpt-4o-2024-05-13'; 3C3H Score 0.7398 (95% CI 0.699446-0.778155, n=267); judge claude-3.5-sonnet; 267/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7747,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-3-5-sonnet-20241022'; 3C3H Score 0.7747 (95% CI 0.735957-0.810855, n=265); judge claude-3.5-sonnet; 265/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7449,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'claude-opus-4-20250514'; 3C3H Score 0.7449 (95% CI 0.704205-0.78277, n=265); judge claude-3.5-sonnet; 265/279 entries answered. No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.708,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'gpt-4o-2024-08-06'; 'Ar Prompt-lvl' 0.708 (Ar instruction-level 0.78, En prompt-level 0.794 not imported). No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.681,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'gpt-4o-mini-2024-07-18'; 'Ar Prompt-lvl' 0.681 (Ar instruction-level 0.747, En prompt-level 0.769 not imported). No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.537,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'claude-3-haiku'; 'Ar Prompt-lvl' 0.537 (Ar instruction-level 0.642, En prompt-level 0.641 not imported). No run date in the file; date is the pinned Space revision date. Entry uses the unversioned alias 'claude-3-haiku'; Anthropic has published only one Claude 3 Haiku snapshot (claude-3-haiku-20240307).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.671,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'claude-3.5-haiku'; 'Ar Prompt-lvl' 0.671 (Ar instruction-level 0.747, En prompt-level 0.782 not imported). No run date in the file; date is the pinned Space revision date. Entry uses the unversioned alias 'claude-3.5-haiku'; Anthropic has published only one Claude 3.5 Haiku snapshot (claude-3-5-haiku-20241022).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.564,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'CohereForAI/c4ai-command-r7b-12-2024'; 'Ar Prompt-lvl' 0.564 (Ar instruction-level 0.652, En prompt-level 0.749 not imported). No run date in the file; date is the pinned Space revision date.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "judgemark-v2",
    "score": 0.4405,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'gpt-4o-mini'; judgemark_score 44.05 (calibrated, 0-100), raw 44.35, Kendall tau 0.628. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100. Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18).",
    "raw_value": 44.05,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "judgemark-v2",
    "score": 0.7764,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'claude-3.5-haiku-20241022'; judgemark_score 77.64 (calibrated, 0-100), raw 76.88, Kendall tau 0.898. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 77.64,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "judgemark-v2",
    "score": 0.7884,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'claude-3.5-sonnet-20240620'; judgemark_score 78.84 (calibrated, 0-100), raw 74.72, Kendall tau 0.905. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 78.84,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "judgemark-v2",
    "score": 0.5542,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'microsoft/wizardlm-2-8x22b'; judgemark_score 55.42 (calibrated, 0-100), raw 52.86, Kendall tau 0.749. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 55.42,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "judgemark-v2",
    "score": 0.8251,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'gpt-4o-2024-11-20'; judgemark_score 82.51 (calibrated, 0-100), raw 77.75, Kendall tau 0.89. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 82.51,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "mistral-small-3",
    "bench": "judgemark-v2",
    "score": 0.6364,
    "source": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Judgemark v2 results/scores.csv (committed 2025-01-31); judge 'mistralai/mistral-small-24b-instruct-2501'; judgemark_score 63.64 (calibrated, 0-100), raw 61.23, Kendall tau 0.808. Measures how well the model scores creative writing against reference quality ordering; composite index divided by 100.",
    "raw_value": 63.64,
    "scale_divisor": 100,
    "source_id": "judgemark-v2-results"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "click-korean-kim",
    "score": 0.6121,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-nano'. gpt-4.1-nano 2025-04-14 model version. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1215/1985 correct after excluding 0 FAILED rows from 1985 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "haerae-korean-kim",
    "score": 0.6795,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-nano'. gpt-4.1-nano 2025-04-14 model version. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1045/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "kmmlu-hard-kim",
    "score": 0.2434,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4.1-nano'. gpt-4.1-nano 2025-04-14 model version. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 999/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "click-korean-kim",
    "score": 0.8046,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o'. GPT-4o 2024-05-13 model version (per the RESULTS.md model-version list). Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1589/1975 correct after excluding 0 FAILED rows from 1975 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "haerae-korean-kim",
    "score": 0.857,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o'. GPT-4o 2024-05-13 model version (per the RESULTS.md model-version list). Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1318/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "kmmlu-hard-kim",
    "score": 0.3962,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o'. GPT-4o 2024-05-13 model version (per the RESULTS.md model-version list). Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1618/4084 correct after excluding 0 FAILED rows from 4084 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "click-korean-kim",
    "score": 0.685,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o-mini'. GPT-4o-mini 2024-07-18 model version. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1346/1965 correct after excluding 0 FAILED rows from 1965 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "haerae-korean-kim",
    "score": 0.764,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o-mini'. GPT-4o-mini 2024-07-18 model version. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1175/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "kmmlu-hard-kim",
    "score": 0.2456,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4o-mini'. GPT-4o-mini 2024-07-18 model version. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1008/4104 correct after excluding 0 FAILED rows from 4104 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "click-korean-kim",
    "score": 0.7282,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4-turbo'. GPT-4-turbo 2024-04-09 model version. Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1431/1965 correct after excluding 0 FAILED rows from 1965 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "haerae-korean-kim",
    "score": 0.7776,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4-turbo'. GPT-4-turbo 2024-04-09 model version. Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1196/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "kmmlu-hard-kim",
    "score": 0.3056,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-4-turbo'. GPT-4-turbo 2024-04-09 model version. Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1251/4094 correct after excluding 0 FAILED rows from 4094 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "click-korean-kim",
    "score": 0.5098,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-3.5-turbo'. GPT-3.5-turbo 2023-06-13 model version (gpt-3.5-turbo-0613). Personal independent run; CLIcK, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 1017/1995 correct after excluding 0 FAILED rows from 1995 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "haerae-korean-kim",
    "score": 0.5267,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-3.5-turbo'. GPT-3.5-turbo 2023-06-13 model version (gpt-3.5-turbo-0613). Personal independent run; HAE_RAE_BENCH 1.0, Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 810/1538 correct after excluding 0 FAILED rows from 1538 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "kmmlu-hard-kim",
    "score": 0.2097,
    "source": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md",
    "by": "independent",
    "note": "Published entry 'GPT-3.5-turbo'. GPT-3.5-turbo 2023-06-13 model version (gpt-3.5-turbo-0613). Personal independent run; KMMLU-HARD (0-shot), Overall accuracy, zero-shot. As-published denominator may exclude filtered/empty responses; this is not guaranteed full-test accuracy. Docs give legacy max_tokens=512/temperature=0.01 but discuss larger budgets for reasoning; per-run budget is not consistently recorded, so those defaults are not asserted for these rows. No evaluation date inferred from model-version dates. Weight/backend revision digest unreported. Only one reasoning configuration retained; full KMMLU and 5-shot duplicates excluded. CSV cross-check: 848/4044 correct after excluding 0 FAILED rows from 4044 recorded rows; matches the table to its published two-decimal precision.",
    "source_id": "korean-personal-benchmark-runs"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "libra-mini-2605",
    "score": 0.282617,
    "source": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard/blob/f5b4b2a59131f86a74af400036ff31b0aae74794/results/microsoft_Phi-3.5-mini-instruct.json",
    "by": "independent",
    "date": "2026-06-08",
    "note": "LIBRA leaderboard results file microsoft_Phi-3.5-mini-instruct.json; LIBRA Mini = mean of the six Mini dataset_total_score values (matreshka_names 0.3740, ru_sci_passage_count 0.0830, ru_2wikimultihopqa 0.3050, long_context_multiq 0.3492, librusec_mhqa 0.3070, ru_babilong_qa3 0.2775) = 0.2826, as computed by the Space's Mini table. The file's own 'total_score' (0.4884) is not used: for Qwen3.5 files it matches neither the 18-dataset nor the Mini mean. Date is the pinned Space revision date.",
    "components": {
      "matreshka_names": 0.374,
      "ru_sci_passage_count": 0.083,
      "ru_2wikimultihopqa": 0.305,
      "long_context_multiq": 0.3492,
      "librusec_mhqa": 0.307,
      "ru_babilong_qa3": 0.2775
    },
    "source_id": "libra-leaderboard"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.344,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-Coder-V2-Instruct'; model_name=deepseek-ai/DeepSeek-Coder-V2-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=34.4186046512%; website rounding=34.4%; divided by 100. Exact upstream model_name deepseek-ai/DeepSeek-Coder-V2-Instruct (the original June 2024 release, catalogued as DeepSeek-Coder-V2 Instruct (0614)); the separate -0724 entry is not merged. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "DeepSeek-Coder-V2-Instruct"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 34.41860465116279,
      "published_display_percent": 34.4,
      "scale_divisor": 100,
      "provenance_id": "deepseek-coder-v2/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "deepseek-ai/DeepSeek-Coder-V2-Instruct",
      "upstream_model_repr": "DeepSeek-Coder-V2-Instruct",
      "upstream_model_style": "DeepSeekCode",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gpt-4-0613",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.325,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-0613'; model_name=gpt-4-0613. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=32.480620155%; website rounding=32.5%; divided by 100. Exact dated API snapshot gpt-4-0613. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "GPT-4-0613"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 32.48062015503876,
      "published_display_percent": 32.5,
      "scale_divisor": 100,
      "provenance_id": "gpt-4-0613/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4-0613",
      "upstream_model_repr": "GPT-4-0613",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.357,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-Turbo-1106'; model_name=gpt-4-1106-preview. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=35.7364341085%; website rounding=35.7%; divided by 100. Exact dated API snapshot gpt-4-1106-preview. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "GPT-4-Turbo-1106"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 35.736434108527135,
      "published_display_percent": 35.7,
      "scale_divisor": 100,
      "provenance_id": "gpt-4-1106-preview/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4-1106-preview",
      "upstream_model_repr": "GPT-4-Turbo-1106",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.401,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-2024-05-13'; model_name=gpt-4o-2024-05-13. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=40.1291989664%; website rounding=40.1%; divided by 100. Exact dated API snapshot gpt-4o-2024-05-13. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "GPT-4O-2024-05-13"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 40.12919896640827,
      "published_display_percent": 40.1,
      "scale_divisor": 100,
      "provenance_id": "gpt-4o-2024-05-13/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4o-2024-05-13",
      "upstream_model_repr": "GPT-4O-2024-05-13",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "o1-preview",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.525,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'O1-Preview-2024-09-12 (N=1)'; model_name=o1-preview-2024-09-12. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=52.4547803618%; website rounding=52.5%; divided by 100. Exact dated API snapshot o1-preview-2024-09-12 (the only o1-preview snapshot); label '(N=1)' retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "O1-Preview-2024-09-12 (N=1)"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 52.45478036175711,
      "published_display_percent": 52.5,
      "scale_divisor": 100,
      "provenance_id": "o1-preview/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o1-preview-2024-09-12",
      "upstream_model_repr": "O1-Preview-2024-09-12 (N=1)",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "o1-mini",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.672,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'O1-Mini-2024-09-12 (N=1)'; model_name=o1-mini-2024-09-12. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=67.1834625323%; website rounding=67.2%; divided by 100. Exact dated API snapshot o1-mini-2024-09-12 (the only o1-mini snapshot); label '(N=1)' retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "O1-Mini-2024-09-12 (N=1)"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 67.18346253229974,
      "published_display_percent": 67.2,
      "scale_divisor": 100,
      "provenance_id": "o1-mini/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o1-mini-2024-09-12",
      "upstream_model_repr": "O1-Mini-2024-09-12 (N=1)",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.319,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Opus'; model_name=claude-3-opus-20240229. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=31.9379844961%; website rounding=31.9%; divided by 100. Exact dated API snapshot claude-3-opus-20240229. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Claude-3-Opus"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 31.93798449612403,
      "published_display_percent": 31.9,
      "scale_divisor": 100,
      "provenance_id": "claude-3-opus-20240229/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-3-opus-20240229",
      "upstream_model_repr": "Claude-3-Opus",
      "upstream_model_style": "AnthropicMessage",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.25,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Sonnet'; model_name=claude-3-sonnet-20240229. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.0129198966%; website rounding=25%; divided by 100. Exact dated API snapshot claude-3-sonnet-20240229. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Claude-3-Sonnet"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 25.012919896640827,
      "published_display_percent": 25,
      "scale_divisor": 100,
      "provenance_id": "claude-3-sonnet-20240229/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-3-sonnet-20240229",
      "upstream_model_repr": "Claude-3-Sonnet",
      "upstream_model_style": "AnthropicMessage",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.426,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-3.5-Sonnet-20240620'; model_name=claude-3-5-sonnet-20240620. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=42.6098191214%; website rounding=42.6%; divided by 100. Exact dated API snapshot claude-3-5-sonnet-20240620. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Claude-3.5-Sonnet-20240620"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 42.60981912144703,
      "published_display_percent": 42.6,
      "scale_divisor": 100,
      "provenance_id": "claude-3-5-sonnet-20240620/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-3-5-sonnet-20240620",
      "upstream_model_repr": "Claude-3.5-Sonnet-20240620",
      "upstream_model_style": "AnthropicMessage",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.364,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-Pro-1.5-002'; model_name=gemini-1.5-pro-002. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=36.4341085271%; website rounding=36.4%; divided by 100. Exact dated API version gemini-1.5-pro-002. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Gemini-Pro-1.5-002"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 36.434108527131784,
      "published_display_percent": 36.4,
      "scale_divisor": 100,
      "provenance_id": "gemini-1-5-pro-002/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-1.5-pro-002",
      "upstream_model_repr": "Gemini-Pro-1.5-002",
      "upstream_model_style": "Google",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.344,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-Flash-1.5-002'; model_name=gemini-1.5-flash-002. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=34.3669250646%; website rounding=34.4%; divided by 100. Exact dated API version gemini-1.5-flash-002. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "Gemini-Flash-1.5-002"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 34.366925064599485,
      "published_display_percent": 34.4,
      "scale_divisor": 100,
      "provenance_id": "gemini-1-5-flash-002/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-1.5-flash-002",
      "upstream_model_repr": "Gemini-Flash-1.5-002",
      "upstream_model_style": "Google",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.724,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'O1-2024-12-17 (N=1) (Tentative)'; model_name=o1-2024-12-17. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=72.3514211886%; website rounding=72.4%; divided by 100. Exact dated API snapshot o1-2024-12-17; the source itself labels this entry '(N=1) (Tentative)', retained here; reasoning effort not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70",
      "file": "src/mocks/performances_generation.json",
      "model_selector": {
        "model": "O1-2024-12-17 (N=1) (Tentative)"
      },
      "date_filter": {
        "gte": 1704067200000,
        "lte": 1730419200000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 387,
      "raw_mean_percent": 72.3514211886305,
      "published_display_percent": 72.4,
      "scale_divisor": 100,
      "provenance_id": "o1-2024-12-17/livecodebench-official-20240101-20241101",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o1-2024-12-17",
      "upstream_model_repr": "O1-2024-12-17 (N=1) (Tentative)",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.287,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-Turbo-2024-04-09'; model_name=gpt-4-turbo-2024-04-09. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=28.7004405286%; website rounding=28.7%; divided by 100. Exact dated API snapshot gpt-4-turbo-2024-04-09. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "GPT-4-Turbo-2024-04-09"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 28.70044052863436,
      "published_display_percent": 28.7,
      "scale_divisor": 100,
      "provenance_id": "gpt-4-turbo-2024-04-09/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4-turbo-2024-04-09",
      "upstream_model_repr": "GPT-4-Turbo-2024-04-09",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.295,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-2024-08-06'; model_name=gpt-4o-2024-08-06. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=29.5374449339%; website rounding=29.5%; divided by 100. Exact dated API snapshot gpt-4o-2024-08-06. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "GPT-4O-2024-08-06"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 29.537444933920703,
      "published_display_percent": 29.5,
      "scale_divisor": 100,
      "provenance_id": "gpt-4o-2024-08-06/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4o-2024-08-06",
      "upstream_model_repr": "GPT-4O-2024-08-06",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.275,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-mini-2024-07-18'; model_name=gpt-4o-mini-2024-07-18. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=27.4669603524%; website rounding=27.5%; divided by 100. Exact dated API snapshot gpt-4o-mini-2024-07-18. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "GPT-4O-mini-2024-07-18"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 27.46696035242291,
      "published_display_percent": 27.5,
      "scale_divisor": 100,
      "provenance_id": "gpt-4o-mini-2024-07-18/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gpt-4o-mini-2024-07-18",
      "upstream_model_repr": "GPT-4O-mini-2024-07-18",
      "upstream_model_style": "OpenAIChat",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "o3-mini",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.674,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'O3-Mini-2025-01-31 (High)'; model_name=o3-mini-2025-01-31__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=67.4008810573%; website rounding=67.4%; divided by 100. Exact dated API snapshot o3-mini-2025-01-31 with published config o3-mini-2025-01-31__high. High effort selected a priori (Low/Med entries of the same snapshot not imported), not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "O3-Mini-2025-01-31 (High)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 67.40088105726872,
      "published_display_percent": 67.4,
      "scale_divisor": 100,
      "provenance_id": "o3-mini/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "o3-mini-2025-01-31__high",
      "upstream_model_repr": "O3-Mini-2025-01-31 (High)",
      "upstream_model_style": "OpenAIReason",
      "reasoning_effort": "high",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.364,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-3.5-Sonnet-20241022'; model_name=claude-3-5-sonnet-20241022. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=36.3876651982%; website rounding=36.4%; divided by 100. Exact dated API snapshot claude-3-5-sonnet-20241022. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Claude-3.5-Sonnet-20241022"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 36.38766519823788,
      "published_display_percent": 36.4,
      "scale_divisor": 100,
      "provenance_id": "claude-3-5-sonnet-20241022/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-3-5-sonnet-20241022",
      "upstream_model_repr": "Claude-3.5-Sonnet-20241022",
      "upstream_model_style": "Claude3",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.202,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Haiku'; model_name=claude-3-haiku-20240307. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=20.2202643172%; website rounding=20.2%; divided by 100. Exact dated API snapshot claude-3-haiku-20240307. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Claude-3-Haiku"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 20.220264317180618,
      "published_display_percent": 20.2,
      "scale_divisor": 100,
      "provenance_id": "claude-3-haiku-20240307/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-3-haiku-20240307",
      "upstream_model_repr": "Claude-3-Haiku",
      "upstream_model_style": "Claude3",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.559,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-Sonnet-4 (Thinking)'; model_name=claude-sonnet-4-20250514. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=55.9471365639%; website rounding=55.9%; divided by 100. Exact dated API snapshot claude-sonnet-4-20250514 with extended thinking (model_style Claude3Thinking); the no-thinking entry of the same snapshot is not imported. Thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Claude-Sonnet-4 (Thinking)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 55.947136563876654,
      "published_display_percent": 55.9,
      "scale_divisor": 100,
      "provenance_id": "claude-sonnet-4-20250514/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-sonnet-4-20250514",
      "upstream_model_repr": "Claude-Sonnet-4 (Thinking)",
      "upstream_model_style": "Claude3Thinking",
      "reasoning_effort": "thinking",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.566,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Claude-Opus-4 (Thinking)'; model_name=claude-opus-4-20250514. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=56.6079295154%; website rounding=56.6%; divided by 100. Exact dated API snapshot claude-opus-4-20250514 with extended thinking (model_style Claude3Thinking); the no-thinking entry of the same snapshot is not imported. Thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Claude-Opus-4 (Thinking)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 56.607929515418505,
      "published_display_percent": 56.6,
      "scale_divisor": 100,
      "provenance_id": "claude-opus-4-20250514/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "claude-opus-4-20250514",
      "upstream_model_repr": "Claude-Opus-4 (Thinking)",
      "upstream_model_style": "Claude3Thinking",
      "reasoning_effort": "thinking",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.736,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Pro-06-05'; model_name=gemini-2.5-pro-preview-06-05. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=73.5682819383%; website rounding=73.6%; divided by 100. Exact dated preview API id gemini-2.5-pro-preview-06-05 (catalogued separately from GA gemini-2.5-pro). Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Gemini-2.5-Pro-06-05"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 73.568281938326,
      "published_display_percent": 73.6,
      "scale_divisor": 100,
      "provenance_id": "gemini-2-5-pro-preview-06-05/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-2.5-pro-preview-06-05",
      "upstream_model_repr": "Gemini-2.5-Pro-06-05",
      "upstream_model_style": "GeminiThinking",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.718,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Pro-05-06'; model_name=gemini-2.5-pro-preview-05-06. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=71.8061674009%; website rounding=71.8%; divided by 100. Exact dated preview API id gemini-2.5-pro-preview-05-06. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Gemini-2.5-Pro-05-06"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 71.80616740088105,
      "published_display_percent": 71.8,
      "scale_divisor": 100,
      "provenance_id": "gemini-2-5-pro-preview-05-06/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-2.5-pro-preview-05-06",
      "upstream_model_repr": "Gemini-2.5-Pro-05-06",
      "upstream_model_style": "GeminiThinking",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.606,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Flash-04-17'; model_name=gemini-2.5-flash-preview-04-17. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=60.5726872247%; website rounding=60.6%; divided by 100. Exact dated preview API id gemini-2.5-flash-preview-04-17; thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Gemini-2.5-Flash-04-17"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 60.5726872246696,
      "published_display_percent": 60.6,
      "scale_divisor": 100,
      "provenance_id": "gemini-2-5-flash-preview-04-17/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-2.5-flash-preview-04-17",
      "upstream_model_repr": "Gemini-2.5-Flash-04-17",
      "upstream_model_style": "GeminiThinking",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.619,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Flash-05-20'; model_name=gemini-2.5-flash-preview-05-20. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=61.8942731278%; website rounding=61.9%; divided by 100. Exact dated preview API id gemini-2.5-flash-preview-05-20; thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Gemini-2.5-Flash-05-20"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 61.8942731277533,
      "published_display_percent": 61.9,
      "scale_divisor": 100,
      "provenance_id": "gemini-2-5-flash-preview-05-20/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "gemini-2.5-flash-preview-05-20",
      "upstream_model_repr": "Gemini-2.5-Flash-05-20",
      "upstream_model_style": "GeminiThinking",
      "reasoning_effort": null,
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "livecodebench-official-20240801-20250501",
    "score": 0.667,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release; entry 'Grok-3-Mini (High)'; model_name=grok-3-mini-beta_high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=66.7400881057%; website rounding=66.7%; divided by 100. Exact API id grok-3-mini-beta with published config grok-3-mini-beta_high; high effort selected a priori. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.",
    "evidence": {
      "kind": "published-per-problem-json",
      "commit": "f186ab5041d6b768733b8ae8145151c867942134",
      "file": "build/performances_generation.json",
      "model_selector": {
        "model": "Grok-3-Mini (High)"
      },
      "date_filter": {
        "gte": 1722470400000,
        "lte": 1746057600000
      },
      "field": "pass@1",
      "aggregation": "Number(mean(values).toFixed(1)) / 100",
      "problem_count": 454,
      "raw_mean_percent": 66.74008810572687,
      "published_display_percent": 66.7,
      "scale_divisor": 100,
      "provenance_id": "grok-3-mini-beta/livecodebench-official-20240801-20250501",
      "provenance_file": "/data/lcb-sharealike-provenance.json"
    },
    "settings": {
      "metric": "pass@1",
      "scenario": "codegeneration",
      "upstream_model_name": "grok-3-mini-beta_high",
      "upstream_model_repr": "Grok-3-Mini (High)",
      "upstream_model_style": "Grok",
      "reasoning_effort": "high",
      "weight_precision": null,
      "temperature": null,
      "top_p": null,
      "samples_per_problem": null,
      "max_output_tokens": null,
      "evaluation_run_date": null,
      "exact_harness_commit": null,
      "weight_digest": null,
      "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings."
    },
    "licensing": {
      "license": "CC-BY-SA-4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "adapter_license": "CC-BY-SA-4.0",
      "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.",
      "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json",
      "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.",
      "notice_file": "/licenses/lcb-sharealike-NOTICE.txt",
      "adapted_data_file": "/data/lcb-sharealike.json"
    },
    "data_license": "CC-BY-SA-4.0",
    "licensed_artifact": "/data/lcb-sharealike.json",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.73913,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 23/23 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 815721b680bf committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0126_outputs",
      "revision": "306b8e8d1e91c40beb6b2b003adb89e7951bd204",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "0749f5ced2cdb6de56ed27177934da4b66e4268d2bf1bdf5ea72d77efc883e70",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 23,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.515625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision e5b08c0ea5d2 committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.4375,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision e5b08c0ea5d2 committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.757812,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'GPT-5.4-Pro (xhigh)', model_config 'openai/gpt-54-pro' (gpt-5.4-pro, reasoning effort xhigh, Responses API, per configs/models/openai/gpt-54-pro.yaml at eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0226_outputs",
      "revision": "92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "b5a1bbc302aa067bcb4ab1edbfb629042c99671777308c5c4170b3bd1ec6948c",
      "model_config": "openai/gpt-54-pro",
      "problems": 32,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.558333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 29b46187fc95 committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-arxivmath-2026-03",
    "score": 0.525,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0326_outputs/blob/69bc5be7b2280c8521854515a981ed074aaa7b40/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath March 2026 outputs dataset (HF revision 69bc5be7b2280c8521854515a981ed074aaa7b40); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 30/30 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 29b46187fc95 committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0326_outputs",
      "revision": "69bc5be7b2280c8521854515a981ed074aaa7b40",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "31944dded3833868d6d08e902fbec46276201af39f7f0a8b2ecc7084d91d3fcc",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 30,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.5125,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision c1576227de56 committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-arxivmath-2026-04",
    "score": 0.475,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0426_outputs/blob/31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath April 2026 outputs dataset (HF revision 31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 2 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision c1576227de56 committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0426_outputs",
      "revision": "31b49f173ea747a53fb613b8c8f24b6b4bdf6fcf",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "4a3c1bafae45d20d9983f6cd1a07b909f60bac0aa820bac3d702c25c9f91b02d",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 40,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-arxivmath-2026-05",
    "score": 0.558333,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0526_outputs/blob/85f1009458c2d2e1527d3e592d04cf40ee8b3a90/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath May 2026 outputs dataset (HF revision 85f1009458c2d2e1527d3e592d04cf40ee8b3a90); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 40/40 problems, 3 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 63ca0d2f778a committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/arxivmath-0526_outputs",
      "revision": "85f1009458c2d2e1527d3e592d04cf40ee8b3a90",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "d38c07f6d91ebc4b450cc0b3f5cdd1278b0880b0cb58a37ce4dddb2aaec50dcf",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 40,
      "attempts_min": 3,
      "attempts_max": 3
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.133065,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 30cc075f1556 committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-brokenarxiv-2026-02",
    "score": 0.177419,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226_outputs/blob/b1346bc6e4bd465b9c163debae81126b1140bfea/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv February 2026 outputs dataset (HF revision b1346bc6e4bd465b9c163debae81126b1140bfea); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 31/31 problems, 4 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 30cc075f1556 committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0226_outputs",
      "revision": "b1346bc6e4bd465b9c163debae81126b1140bfea",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "633ef0323516d696da53d817e52d956018af70252185aa6768e0c8280baf02cc",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 31,
      "attempts_min": 4,
      "attempts_max": 4
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.151786,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision f580302e137e committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-brokenarxiv-2026-03",
    "score": 0.196429,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326_outputs/blob/d719c23340e971fe956a2503bd157bfdcb67d146/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv March 2026 outputs dataset (HF revision d719c23340e971fe956a2503bd157bfdcb67d146); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 56/56 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision f580302e137e committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0326_outputs",
      "revision": "d719c23340e971fe956a2503bd157bfdcb67d146",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "2b8ce3928734687cfc771bd64e5f3cdec9dc41c650c91730590d05b210598a8e",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 56,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.221311,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'DeepSeek-v4-Pro (Max)', model_config 'deepseek/deepseek_v4_pro' (deepseek-v4-pro (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Pro, per configs/models/deepseek/deepseek_v4_pro.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 27410aa882a2 committed 2026-06-23, before the 0813 GA replaced the April preview under the same API id on 2026-08-13; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "deepseek/deepseek_v4_pro",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-brokenarxiv-2026-04",
    "score": 0.237705,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426_outputs/blob/34779f2c3c287fda88ea06b4c7885a1d10ae8b36/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena BrokenArXiv April 2026 outputs dataset (HF revision 34779f2c3c287fda88ea06b4c7885a1d10ae8b36); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 61/61 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: the same run (identical per-attempt results) is already in dataset revision 27410aa882a2 committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0426_outputs",
      "revision": "34779f2c3c287fda88ea06b4c7885a1d10ae8b36",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "68f10bcf55926258a256f67f3cfb6a557b0844c4745e037bc207f5280d911860",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 61,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "matharena-brokenarxiv-2026-05",
    "score": 0.195,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526_outputs/blob/7ca7ebc694e921b56eb9b5968374918bf2a2b4e4/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-06-23",
    "note": "MathArena BrokenArXiv May 2026 outputs dataset (HF revision 7ca7ebc694e921b56eb9b5968374918bf2a2b4e4); published model_name 'DeepSeek-v4-Flash (Max)', model_config 'deepseek/deepseek_v4_flash' (deepseek-v4-flash (DeepSeek API), reasoning_effort max, config date 2026-04-24; HF deepseek-ai/DeepSeek-V4-Flash, per configs/models/deepseek/deepseek_v4_flash.yaml at eth-sri/matharena b89f2f0ad64c). 50/50 problems, 2 attempts per problem; score = mean normalized judge score (points_judge_1/2; 2 = explicitly recognises the statement is false), averaged per problem then over problems, computed by whatmodel.sh from the 'points_judge_1' column. Competition config has no harness opt-in, so the plain API path was used (no tools). Identity: this pinned revision was committed 2026-06-23, before DeepSeek-V4-Flash-0731 replaced the April preview under the same API id on 2026-07-31; config dated 2026-04-24, so this is the April 2026 preview checkpoint. CC BY-SA 4.0 (share-alike tranche).",
    "evidence": {
      "kind": "per-attempt parquet aggregated",
      "dataset": "MathArena/brokenarxiv-0526_outputs",
      "revision": "7ca7ebc694e921b56eb9b5968374918bf2a2b4e4",
      "file": "data/train-00000-of-00001.parquet",
      "file_sha256": "08ea0c17c25e7ffa06f81fe47abd5d9b50a1515e6bf5ddf6153a75809ba9e59b",
      "model_config": "deepseek/deepseek_v4_flash",
      "problems": 50,
      "attempts_min": 2,
      "attempts_max": 2
    },
    "data_license": "CC-BY-SA-4.0",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "inkling",
    "bench": "mideind-icelandic-average",
    "score": 0.46518,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Inkling' (together_ai/thinkingmachines/Inkling), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"chat_template_kwargs\": {\"reasoning_effort\": \"high\"}}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.329253, Language Proficiency 0.567125). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b. Board note: 'Served by Together at NVFP4; 41B active of 975B (MoE).'",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "mideind-icelandic-average",
    "score": 0.411229,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-08",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-4o' (openai/gpt-4o-2024-08-06), thinking='none', gen_kwargs={\"max_tokens\": 16384}; highest of the listed settings ['none']. Published 'average' (Knowledge 0.25768, Language Proficiency 0.526391). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "inkling-small",
    "bench": "mideind-icelandic-average",
    "score": 0.294788,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Inkling Small' (hosted_vllm/thinkingmachines/Inkling-Small-NVFP4), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"chat_template_kwargs\": {\"reasoning_effort\": \"high\"}}}; highest of the listed settings ['high']. Published 'average' (Knowledge 0.185269, Language Proficiency 0.376928). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b. Board note: 'Self-hosted on vllm at NVFP4 (W4A4, TP1); 12B active of 276B (MoE).'",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "mideind-icelandic-average",
    "score": 0.261531,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Gemini 2.5 Flash Lite' (gemini/gemini-2.5-flash-lite), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.182244, Language Proficiency 0.320995). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "mideind-icelandic-average",
    "score": 0.175943,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-4.1 Nano' (openai/gpt-4.1-nano), thinking='none', gen_kwargs={\"max_tokens\": 32000}; highest of the listed settings ['none']. Published 'average' (Knowledge 0.072182, Language Proficiency 0.253764). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "mideind-icelandic-average",
    "score": 0.129017,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json",
    "by": "independent",
    "date": "2026-09-08",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-3.5 Turbo' (openai/gpt-3.5-turbo-0125), thinking='none', gen_kwargs={\"max_tokens\": 4096}; highest of the listed settings ['none']. Published 'average' (Knowledge 0.083522, Language Proficiency 0.163138). Date = latest benchmark_evaluated_at. Board generated 2026-09-25T11:19:38Z; Space revision f8e42e15da1b985242bbb29eff139ec62405b53b.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "llama-2-70b",
    "bench": "mmlu-pro",
    "score": 0.3628656914893617,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Llama-2-70b-hf_5shots.zip",
    "by": "independent",
    "note": "Published entry 'meta-llama/Llama-2-70b-hf'. Author-released 5-shot CoT predictions, 4366/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Base (pretrained) checkpoint, not the chat model.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "llama-3-8b",
    "bench": "mmlu-pro",
    "score": 0.4011801861702128,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Meta-Llama-3-8B-Instruct_5shots.zip",
    "by": "independent",
    "note": "Published entry 'meta-llama/Meta-Llama-3-8B-Instruct'. Author-released 5-shot CoT predictions, 4827/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "mmlu-pro",
    "score": 0.24152260638297873,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Mistral-7B-Instruct-v0.1_5shots.zip",
    "by": "independent",
    "note": "Published entry 'mistralai/Mistral-7B-Instruct-v0.1'. Author-released 5-shot CoT predictions, 2906/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "mmlu-pro",
    "score": 0.28756648936170215,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Mistral-7B-Instruct-v0.2_5shots.zip",
    "by": "independent",
    "note": "Published entry 'mistralai/Mistral-7B-Instruct-v0.2'. Author-released 5-shot CoT predictions, 3460/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "mmlu-pro",
    "score": 0.41888297872340424,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Mixtral-8x7B-Instruct-v0.1_5shots.zip",
    "by": "independent",
    "note": "Published entry 'mistralai/Mixtral-8x7B-Instruct-v0.1'. Author-released 5-shot CoT predictions, 5040/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "phi-3-mini-4k-instruct",
    "bench": "mmlu-pro",
    "score": 0.4148105053191489,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Phi-3-mini-4k-instruct_5shots.zip",
    "by": "independent",
    "note": "Published entry 'microsoft/Phi-3-mini-4k-instruct'. Author-released 5-shot CoT predictions, 4991/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Outputs first committed 2024-05-21 (commit 1f9b380), before Microsoft's June 2024 update, so the April 2024 checkpoint.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "mmlu-pro",
    "score": 0.36220079787234044,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Qwen1.5-14B-Chat_5shots.zip",
    "by": "independent",
    "note": "Published entry 'Qwen/Qwen1.5-14B-Chat'. Author-released 5-shot CoT predictions, 4358/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "mmlu-pro",
    "score": 0.4714926861702128,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Qwen1.5-72B-Chat_5shots.zip",
    "by": "independent",
    "note": "Published entry 'Qwen/Qwen1.5-72B-Chat'. Author-released 5-shot CoT predictions, 5673/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "mmlu-pro",
    "score": 0.2643783244680851,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_Qwen1.5-7B-Chat_5shots.zip",
    "by": "independent",
    "note": "Published entry 'Qwen/Qwen1.5-7B-Chat'. Author-released 5-shot CoT predictions, 3181/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "command-r-03-2024",
    "bench": "mmlu-pro",
    "score": 0.3659408244680851,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_c4ai-command-r-v01_5shots.zip",
    "by": "independent",
    "note": "Published entry 'CohereForAI/c4ai-command-r-v01'. Author-released 5-shot CoT predictions, 4403/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. The March 2024 Command R weights (the 08-2024 release is a separate repo).",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "mmlu-pro",
    "score": 0.6211768617021277,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_claude-3-5-haiku-20241022_5shots.json.zip",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. Author-released 5-shot CoT predictions, 7474/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "mmlu-pro",
    "score": 0.7764295212765957,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_claude-3-5-sonnet-20241022_5shots.json.zip",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. Author-released 5-shot CoT predictions, 9342/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mmlu-pro",
    "score": 0.7610538563829787,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_claude-3.5-sonnet_5shots.zip",
    "by": "independent",
    "note": "Published entry 'claude-3.5-sonnet'. Author-released 5-shot CoT predictions, 9157/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12030 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Undated label, but the archive was committed 2024-10-17, before claude-3-5-sonnet-20241022 existed, so the only Claude 3.5 Sonnet snapshot then was 20240620.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "mmlu-pro",
    "score": 0.6583277925531915,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_deepseek-chat-v2_5_5shots.zip",
    "by": "independent",
    "note": "Published entry 'deepseek-chat-v2.5'. Author-released 5-shot CoT predictions, 7921/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Outputs first committed 2024-09-10 ('add deepseek v2.5'), after the 2024-09-05 DeepSeek-V2.5 release and before V2.5-1210.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "mmlu-pro",
    "score": 0.639876994680851,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gemini-1.5-flash-002_5shots.zip",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-002'. Author-released 5-shot CoT predictions, 7699/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12012 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "mmlu-pro",
    "score": 0.7017952127659575,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gemini-1.5-pro-002_5shots.zip",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-002'. Author-released 5-shot CoT predictions, 8444/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12020 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "mmlu-pro",
    "score": 0.9105718085106383,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gemini-3.1-pro_5-shots.zip",
    "by": "independent",
    "note": "Published entry 'gemini-3.1-pro'. Author-released 5-shot CoT predictions, 10956/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12020 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Per-category files plus summary.json; committed 2026-02-20 ('add gemini-3.1-pro detailed eval result'), i.e. the Gemini 3.1 Pro preview; thinking level not stated.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "mmlu-pro",
    "score": 0.7467586436170213,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gpt-4o-2024-08-06_5shots.zip",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. Author-released 5-shot CoT predictions, 8985/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu-pro",
    "score": 0.6309009308510638,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gpt-4o-mini_5shots.zip",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini'. Author-released 5-shot CoT predictions, 7591/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18).",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mmlu-pro",
    "score": 0.7206615691489362,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_gpt4o(2024-05-13)_5shots.zip",
    "by": "independent",
    "note": "Published entry 'gpt4o(2024-05-13)'. Author-released 5-shot CoT predictions, 8671/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "mmlu-pro",
    "score": 0.49459773936170215,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_jamba-1.5-large_5shots.zip",
    "by": "independent",
    "note": "Published entry 'jamba-1.5-large'. Author-released 5-shot CoT predictions, 5951/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.047181,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_Claude-3.5-Sonnet(Oct)' (metadata verified: true) in each of the seven language 'verified' splits; resolved 77/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated. Folder label 'Claude-3.5-Sonnet(Oct)' = the October 2024 snapshot claude-3-5-sonnet-20241022.",
    "source_id": "multi-swe-bench-experiments"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.050858,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_Claude-3.7-Sonnet' (metadata verified: true) in each of the seven language 'verified' splits; resolved 83/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_Claude-3.7-Sonnet/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated. Only one Claude 3.7 Sonnet snapshot (20250219) exists; thinking not stated.",
    "source_id": "multi-swe-bench-experiments"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.03799,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_GPT-4o-1120' (metadata verified: true) in each of the seven language 'verified' splits; resolved 62/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_GPT-4o-1120/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated. Folder label 'GPT-4o-1120' = gpt-4o-2024-11-20.",
    "source_id": "multi-swe-bench-experiments"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.064338,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_OpenAI-o1' (metadata verified: true) in each of the seven language 'verified' splits; resolved 105/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_OpenAI-o1/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated. Unversioned 'o1'; o1-2024-12-17 is the only o1 snapshot; reasoning effort not stated.",
    "source_id": "multi-swe-bench-experiments"
  },
  {
    "model": "o3-mini",
    "bench": "multi-swe-bench-7lang-magentless",
    "score": 0.036152,
    "source": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Multi-SWE-bench experiments, submission folder '20250329_MagentLess_OpenAI-o3-mini-high' (metadata verified: true) in each of the seven language 'verified' splits; resolved 59/1632 summed from evaluation/<lang>/verified/20250329_MagentLess_OpenAI-o3-mini-high/results/results.json. Agent scaffold MagentLess; the MopenHands and MSWE-agent runs of the same models are not imported to avoid duplicating one study. Serving provider and precision not stated. o3-mini-2025-01-31 (only snapshot) at reasoning effort high.",
    "source_id": "multi-swe-bench-experiments"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.762838,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4o-2024-08-06' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 76.2838 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 76.28381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 90.82,
      "GSM8K": 73.1,
      "DC-Homograph": 87.04,
      "MC-Homograph": 95.62,
      "PiQA": 95.1,
      "Proverb-Quiz": 96.76,
      "VerbEval": 85.89,
      "Winogrande": 86.18,
      "Arc-Challenge": 95.09,
      "Arc-Easy": 97.22,
      "Feqh": 46.86,
      "Hallucination (Truthfulness)": 74.64,
      "P-Hellaswag": 85.53,
      "Law": 47.67,
      "AUT Multiple Choice": 67.7,
      "Parsi Literature": 45.95,
      "BoolQA": 94.1,
      "Reading Comprehension": 55.34,
      "PartExpert": 57.36,
      "MMLU Pro": 47.1,
      "Iranian Social Norms": 76.89
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.738448,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'google__gemini-2.0-flash-001' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 73.8448 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 73.844762,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 87.76,
      "GSM8K": 53.7,
      "DC-Homograph": 79.63,
      "MC-Homograph": 91.71,
      "PiQA": 90.59,
      "Proverb-Quiz": 95.14,
      "VerbEval": 85.15,
      "Winogrande": 78.74,
      "Arc-Challenge": 91.35,
      "Arc-Easy": 97.22,
      "Feqh": 53.14,
      "Hallucination (Truthfulness)": 68.87,
      "P-Hellaswag": 82.95,
      "Law": 45.67,
      "AUT Multiple Choice": 60.9,
      "Parsi Literature": 44.02,
      "BoolQA": 91.3,
      "Reading Comprehension": 67.92,
      "PartExpert": 59.5,
      "MMLU Pro": 47.8,
      "Iranian Social Norms": 77.68
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.700352,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4-turbo-2024-04-09' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 70.0352 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 70.035238,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 80.87,
      "GSM8K": 30.6,
      "DC-Homograph": 88.89,
      "MC-Homograph": 93.78,
      "PiQA": 94.19,
      "Proverb-Quiz": 86.76,
      "VerbEval": 74.29,
      "Winogrande": 83.08,
      "Arc-Challenge": 91.35,
      "Arc-Easy": 96.47,
      "Feqh": 40,
      "Hallucination (Truthfulness)": 55.69,
      "P-Hellaswag": 80.68,
      "Law": 42,
      "AUT Multiple Choice": 62.6,
      "Parsi Literature": 40.93,
      "BoolQA": 92.2,
      "Reading Comprehension": 67.17,
      "PartExpert": 53.18,
      "MMLU Pro": 40.1,
      "Iranian Social Norms": 75.91
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.688938,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4o-mini-2024-07-18' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 68.8938 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 68.89381,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 79.08,
      "GSM8K": 60.9,
      "DC-Homograph": 68.52,
      "MC-Homograph": 90.09,
      "PiQA": 90.89,
      "Proverb-Quiz": 84.05,
      "VerbEval": 74.23,
      "Winogrande": 75.73,
      "Arc-Challenge": 86.43,
      "Arc-Easy": 94.01,
      "Feqh": 41.71,
      "Hallucination (Truthfulness)": 82.04,
      "P-Hellaswag": 83.84,
      "Law": 34,
      "AUT Multiple Choice": 54.8,
      "Parsi Literature": 40.93,
      "BoolQA": 93.3,
      "Reading Comprehension": 63.29,
      "PartExpert": 42.54,
      "MMLU Pro": 34.8,
      "Iranian Social Norms": 71.59
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.648067,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Mistral-Large-Instruct-2407' (query id mistralai/Mistral-Large-Instruct-2407, precision BF16); Average Accuracy = mean of the 21 published task scores = 64.8067 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 64.806667,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 72.45,
      "GSM8K": 35.3,
      "DC-Homograph": 73.15,
      "MC-Homograph": 93.09,
      "PiQA": 88.29,
      "Proverb-Quiz": 65.95,
      "VerbEval": 53.81,
      "Winogrande": 74.93,
      "Arc-Challenge": 89,
      "Arc-Easy": 94.87,
      "Feqh": 26.86,
      "Hallucination (Truthfulness)": 61.28,
      "P-Hellaswag": 84.64,
      "Law": 36,
      "AUT Multiple Choice": 55.4,
      "Parsi Literature": 37.71,
      "BoolQA": 87.3,
      "Reading Comprehension": 67.9,
      "PartExpert": 48.62,
      "MMLU Pro": 40,
      "Iranian Social Norms": 74.39
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.629662,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-r-plus-08-2024' (query id CohereLabs/c4ai-command-r-plus-08-2024, precision F16); Average Accuracy = mean of the 21 published task scores = 62.9662 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 62.96619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 73.21,
      "GSM8K": 26,
      "DC-Homograph": 62.96,
      "MC-Homograph": 92.86,
      "PiQA": 86.29,
      "Proverb-Quiz": 82.97,
      "VerbEval": 63.18,
      "Winogrande": 68.02,
      "Arc-Challenge": 80.34,
      "Arc-Easy": 91.34,
      "Feqh": 33.71,
      "Hallucination (Truthfulness)": 54.39,
      "P-Hellaswag": 81.26,
      "Law": 40,
      "AUT Multiple Choice": 55.1,
      "Parsi Literature": 37.45,
      "BoolQA": 84,
      "Reading Comprehension": 64.9,
      "PartExpert": 44.08,
      "MMLU Pro": 29.8,
      "Iranian Social Norms": 70.43
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "aya-expanse-32b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.629052,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'aya-expanse-32b' (query id CohereLabs/aya-expanse-32b, precision F16); Average Accuracy = mean of the 21 published task scores = 62.9052 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 62.905238,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 73.72,
      "GSM8K": 17.5,
      "DC-Homograph": 62.96,
      "MC-Homograph": 87.56,
      "PiQA": 91.19,
      "Proverb-Quiz": 77.03,
      "VerbEval": 61.95,
      "Winogrande": 70.5,
      "Arc-Challenge": 85.15,
      "Arc-Easy": 93.37,
      "Feqh": 37.14,
      "Hallucination (Truthfulness)": 44.84,
      "P-Hellaswag": 81.7,
      "Law": 38.67,
      "AUT Multiple Choice": 54.7,
      "Parsi Literature": 34.75,
      "BoolQA": 89.7,
      "Reading Comprehension": 67.25,
      "PartExpert": 44.29,
      "MMLU Pro": 32.1,
      "Iranian Social Norms": 74.94
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.618886,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'QwQ-32B-Preview' (query id Qwen/QwQ-32B-Preview, precision BF16); Average Accuracy = mean of the 21 published task scores = 61.8886 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 61.888571,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 63.27,
      "GSM8K": 34.7,
      "DC-Homograph": 61.11,
      "MC-Homograph": 88.25,
      "PiQA": 81.28,
      "Proverb-Quiz": 58.11,
      "VerbEval": 51.97,
      "Winogrande": 75.64,
      "Arc-Challenge": 85.58,
      "Arc-Easy": 91.44,
      "Feqh": 41.14,
      "Hallucination (Truthfulness)": 38.84,
      "P-Hellaswag": 84.13,
      "Law": 43,
      "AUT Multiple Choice": 50.6,
      "Parsi Literature": 39.77,
      "BoolQA": 88.5,
      "Reading Comprehension": 65.38,
      "PartExpert": 47.39,
      "MMLU Pro": 37.3,
      "Iranian Social Norms": 72.26
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "qwq-32b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.605562,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'QwQ-32B' (query id Qwen/QwQ-32B, precision BF16); Average Accuracy = mean of the 21 published task scores = 60.5562 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 60.55619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 60.71,
      "GSM8K": 29.3,
      "DC-Homograph": 58.33,
      "MC-Homograph": 88.25,
      "PiQA": 81.68,
      "Proverb-Quiz": 59.19,
      "VerbEval": 52.31,
      "Winogrande": 73.07,
      "Arc-Challenge": 84.94,
      "Arc-Easy": 90.8,
      "Feqh": 41.71,
      "Hallucination (Truthfulness)": 48.93,
      "P-Hellaswag": 82.22,
      "Law": 38,
      "AUT Multiple Choice": 49.3,
      "Parsi Literature": 37.71,
      "BoolQA": 88.5,
      "Reading Comprehension": 50.25,
      "PartExpert": 46.75,
      "MMLU Pro": 39,
      "Iranian Social Norms": 70.73
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.601995,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-r-plus' (query id CohereLabs/c4ai-command-r-plus, precision F16); Average Accuracy = mean of the 21 published task scores = 60.1995 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 60.199524,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 69.9,
      "GSM8K": 20.9,
      "DC-Homograph": 62.96,
      "MC-Homograph": 91.24,
      "PiQA": 85.49,
      "Proverb-Quiz": 80.27,
      "VerbEval": 55.68,
      "Winogrande": 69.97,
      "Arc-Challenge": 77.14,
      "Arc-Easy": 89.41,
      "Feqh": 33.14,
      "Hallucination (Truthfulness)": 27.45,
      "P-Hellaswag": 80.46,
      "Law": 40,
      "AUT Multiple Choice": 49.5,
      "Parsi Literature": 35.14,
      "BoolQA": 87.7,
      "Reading Comprehension": 67.08,
      "PartExpert": 42.59,
      "MMLU Pro": 27.8,
      "Iranian Social Norms": 70.37
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.600714,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-4.1-nano-2025-04-14' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 60.0714 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 60.071429,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 68.11,
      "GSM8K": 58.4,
      "DC-Homograph": 49.07,
      "MC-Homograph": 78.11,
      "PiQA": 84.58,
      "Proverb-Quiz": 67.84,
      "VerbEval": 66.21,
      "Winogrande": 60.32,
      "Arc-Challenge": 81.41,
      "Arc-Easy": 91.55,
      "Feqh": 32,
      "Hallucination (Truthfulness)": 51.24,
      "P-Hellaswag": 77.96,
      "Law": 32.67,
      "AUT Multiple Choice": 46.1,
      "Parsi Literature": 36.42,
      "BoolQA": 81.7,
      "Reading Comprehension": 50.66,
      "PartExpert": 42.49,
      "MMLU Pro": 29.9,
      "Iranian Social Norms": 74.76
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "command-r-03-2024",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.553519,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-r-v01' (query id CohereLabs/c4ai-command-r-v01, precision F16); Average Accuracy = mean of the 21 published task scores = 55.3519 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 55.351905,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 66.07,
      "GSM8K": 8.4,
      "DC-Homograph": 50,
      "MC-Homograph": 83.18,
      "PiQA": 84.38,
      "Proverb-Quiz": 71.35,
      "VerbEval": 45.42,
      "Winogrande": 61.2,
      "Arc-Challenge": 75.11,
      "Arc-Easy": 86.31,
      "Feqh": 25.71,
      "Hallucination (Truthfulness)": 45.51,
      "P-Hellaswag": 79.28,
      "Law": 36.67,
      "AUT Multiple Choice": 46.8,
      "Parsi Literature": 32.18,
      "BoolQA": 82.1,
      "Reading Comprehension": 64.52,
      "PartExpert": 39.08,
      "MMLU Pro": 27.9,
      "Iranian Social Norms": 51.22
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.552057,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-30B-A3B' (query id Qwen/Qwen3-30B-A3B, precision BF16); Average Accuracy = mean of the 21 published task scores = 55.2057 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Original April 2025 hybrid checkpoint.",
    "raw_value": 55.205714,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 65.05,
      "GSM8K": 28.8,
      "DC-Homograph": 57.41,
      "MC-Homograph": 86.41,
      "PiQA": 72.47,
      "Proverb-Quiz": 50.81,
      "VerbEval": 48.09,
      "Winogrande": 65.28,
      "Arc-Challenge": 87.39,
      "Arc-Easy": 93.58,
      "Feqh": 23.43,
      "Hallucination (Truthfulness)": 3.54,
      "P-Hellaswag": 83.1,
      "Law": 35.33,
      "AUT Multiple Choice": 48,
      "Parsi Literature": 36.55,
      "BoolQA": 86.2,
      "Reading Comprehension": 66.24,
      "PartExpert": 41.13,
      "MMLU Pro": 36.3,
      "Iranian Social Norms": 44.21
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "aya-expanse-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.535767,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'aya-expanse-8b' (query id CohereLabs/aya-expanse-8b, precision F16); Average Accuracy = mean of the 21 published task scores = 53.5767 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 53.576667,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 58.67,
      "GSM8K": 9.8,
      "DC-Homograph": 51.85,
      "MC-Homograph": 80.65,
      "PiQA": 80.18,
      "Proverb-Quiz": 60,
      "VerbEval": 48.06,
      "Winogrande": 64.04,
      "Arc-Challenge": 71.47,
      "Arc-Easy": 84.6,
      "Feqh": 29.71,
      "Hallucination (Truthfulness)": 23.52,
      "P-Hellaswag": 76.49,
      "Law": 32.33,
      "AUT Multiple Choice": 45.8,
      "Parsi Literature": 34.49,
      "BoolQA": 82.3,
      "Reading Comprehension": 61.98,
      "PartExpert": 35.56,
      "MMLU Pro": 21.9,
      "Iranian Social Norms": 71.71
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.506786,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-3.5-turbo-0125' (query id None, precision unknown); Average Accuracy = mean of the 21 published task scores = 50.6786 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 50.678571,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 55.1,
      "GSM8K": 18.9,
      "DC-Homograph": 50.93,
      "MC-Homograph": 73.04,
      "PiQA": 70.87,
      "Proverb-Quiz": 48.65,
      "VerbEval": 47.75,
      "Winogrande": 55.27,
      "Arc-Challenge": 60.04,
      "Arc-Easy": 76.68,
      "Feqh": 27.43,
      "Hallucination (Truthfulness)": 39.93,
      "P-Hellaswag": 66.94,
      "Law": 30,
      "AUT Multiple Choice": 40.9,
      "Parsi Literature": 32.3,
      "BoolQA": 77.3,
      "Reading Comprehension": 63.17,
      "PartExpert": 35.28,
      "MMLU Pro": 23.4,
      "Iranian Social Norms": 70.37
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "qwen3-4b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.504762,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen3-4B' (query id Qwen/Qwen3-4B, precision BF16); Average Accuracy = mean of the 21 published task scores = 50.4762 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Original April 2025 hybrid checkpoint.",
    "raw_value": 50.47619,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 43.88,
      "GSM8K": 20.1,
      "DC-Homograph": 38.89,
      "MC-Homograph": 76.27,
      "PiQA": 66.07,
      "Proverb-Quiz": 45.41,
      "VerbEval": 41.23,
      "Winogrande": 54.56,
      "Arc-Challenge": 73.61,
      "Arc-Easy": 83.42,
      "Feqh": 30.29,
      "Hallucination (Truthfulness)": 25.29,
      "P-Hellaswag": 78.03,
      "Law": 30.33,
      "AUT Multiple Choice": 40.6,
      "Parsi Literature": 31.79,
      "BoolQA": 81.9,
      "Reading Comprehension": 63.43,
      "PartExpert": 37.28,
      "MMLU Pro": 28.9,
      "Iranian Social Norms": 68.72
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.50369,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'c4ai-command-r7b-12-2024' (query id CohereLabs/c4ai-command-r7b-12-2024, precision BF16); Average Accuracy = mean of the 21 published task scores = 50.3690 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 50.369048,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 53.06,
      "GSM8K": 9.6,
      "DC-Homograph": 47.22,
      "MC-Homograph": 71.43,
      "PiQA": 75.98,
      "Proverb-Quiz": 56.22,
      "VerbEval": 42.52,
      "Winogrande": 58.99,
      "Arc-Challenge": 68.7,
      "Arc-Easy": 81.28,
      "Feqh": 26.29,
      "Hallucination (Truthfulness)": 42.34,
      "P-Hellaswag": 72.3,
      "Law": 31.33,
      "AUT Multiple Choice": 44.6,
      "Parsi Literature": 32.3,
      "BoolQA": 67.3,
      "Reading Comprehension": 61.38,
      "PartExpert": 35.53,
      "MMLU Pro": 23.1,
      "Iranian Social Norms": 56.28
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.455495,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'gpt-oss-20b' (query id openai/gpt-oss-20b, precision BF16); Average Accuracy = mean of the 21 published task scores = 45.5495 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Note: the separate 'gpt-oss-120b' row lists the same query id openai/gpt-oss-20b and is not imported.",
    "raw_value": 45.549524,
    "scale_divisor": 100,
    "components": {
      "GeneralKnowledge": 54.85,
      "GSM8K": 9,
      "DC-Homograph": 61.11,
      "MC-Homograph": 61.29,
      "PiQA": 68.17,
      "Proverb-Quiz": 34.86,
      "VerbEval": 52.56,
      "Winogrande": 55.27,
      "Arc-Challenge": 70.19,
      "Arc-Easy": 83.32,
      "Feqh": 28.57,
      "Hallucination (Truthfulness)": 45.6,
      "P-Hellaswag": 46.22,
      "Law": 28.67,
      "AUT Multiple Choice": 42,
      "Parsi Literature": 30.24,
      "BoolQA": 64,
      "Reading Comprehension": 12.62,
      "PartExpert": 37.06,
      "MMLU Pro": 19.9,
      "Iranian Social Norms": 51.04
    },
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-exp-03-25",
    "bench": "open-pt-llm-avg9",
    "score": 0.8837,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Gemini 2.5 Pro Experimental [reasoning] (0325) [2025-04-03]' (source=external, precision=?); published average 88.37 (divided by 100). API run (external); gemini-2.5-pro-exp-03-25, reasoning (default thinking). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 88.37,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 97.69,
      "bluex": 94.99,
      "oab_exams": 92.16,
      "assin2_rte": 94.16,
      "assin2_sts": 83.78,
      "faquad_nli": 87.39,
      "hatebr_offensive": 92.48,
      "portuguese_hate_speech": 73.36,
      "tweetsentbr": 79.28
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "open-pt-llm-avg9",
    "score": 0.8449,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-04",
    "note": "Open PT-LLM Leaderboard row 'Claude 3.7 Sonnet (2025-02-19) [2025-04-04]' (source=external, precision=?); published average 84.49 (divided by 100). API run (external); dated snapshot claude-3-7-sonnet-20250219; thinking not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 84.49,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 89.01,
      "bluex": 84.56,
      "oab_exams": 83.55,
      "assin2_rte": 94.73,
      "assin2_sts": 80.88,
      "faquad_nli": 80.98,
      "hatebr_offensive": 91.25,
      "portuguese_hate_speech": 76.99,
      "tweetsentbr": 78.42
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "open-pt-llm-avg9",
    "score": 0.8382,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-09",
    "note": "Open PT-LLM Leaderboard row 'GPT-4o (2024-08-06) [2025-04-09]' (source=external, precision=?); published average 83.82 (divided by 100). API run (external); dated snapshot gpt-4o-2024-08-06. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 83.82,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 85.3,
      "bluex": 79.69,
      "oab_exams": 82,
      "assin2_rte": 94.07,
      "assin2_sts": 80.79,
      "faquad_nli": 86.54,
      "hatebr_offensive": 93.2,
      "portuguese_hate_speech": 75.13,
      "tweetsentbr": 77.61
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "open-pt-llm-avg9",
    "score": 0.823,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Gemini 2.0 Flash (001) [2025-04-03]' (source=external, precision=?); published average 82.3 (divided by 100). API run (external); versioned gemini-2.0-flash-001. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 82.3,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 87.89,
      "bluex": 80.39,
      "oab_exams": 77.68,
      "assin2_rte": 93.05,
      "assin2_sts": 84.4,
      "faquad_nli": 75.34,
      "hatebr_offensive": 88.9,
      "portuguese_hate_speech": 76.55,
      "tweetsentbr": 76.53
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.8086,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-VL-32B-Instruct (API) [2025-04-03]' (source=external, precision=?); published average 80.86 (divided by 100). API run (external) of Qwen2.5-VL-32B-Instruct (text-only tasks). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 80.86,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 86,
      "bluex": 80.53,
      "oab_exams": 68.88,
      "assin2_rte": 94.73,
      "assin2_sts": 77.81,
      "faquad_nli": 84.47,
      "hatebr_offensive": 88.11,
      "portuguese_hate_speech": 76.96,
      "tweetsentbr": 70.27
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "open-pt-llm-avg9",
    "score": 0.799,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Gemini 1.5 Flash (002) [2025-04-03]' (source=external, precision=?); published average 79.9 (divided by 100). API run (external); versioned gemini-1.5-flash-002. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.9,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 83.28,
      "bluex": 76.08,
      "oab_exams": 63.69,
      "assin2_rte": 94.12,
      "assin2_sts": 83.8,
      "faquad_nli": 83.61,
      "hatebr_offensive": 90.46,
      "portuguese_hate_speech": 74.06,
      "tweetsentbr": 69.98
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "open-pt-llm-avg9",
    "score": 0.7932,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-09",
    "note": "Open PT-LLM Leaderboard row 'Gemini 1.5 Flash (001) [2024-08-09]' (source=external, precision=?); published average 79.32 (divided by 100). API run (external); versioned gemini-1.5-flash-001. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.32,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 83.07,
      "bluex": 75.8,
      "oab_exams": 64.46,
      "assin2_rte": 93.66,
      "assin2_sts": 83.88,
      "faquad_nli": 79.64,
      "hatebr_offensive": 90.92,
      "portuguese_hate_speech": 69.33,
      "tweetsentbr": 73.13
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "open-pt-llm-avg9",
    "score": 0.7928,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.2-90B-Vision-Instruct (Vertex AI) [2025-04-03]' (source=external, precision=?); published average 79.28 (divided by 100). API run on Vertex AI (external); the hub row is 4-bit and not imported. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.28,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 82.16,
      "bluex": 74.83,
      "oab_exams": 70.62,
      "assin2_rte": 92.17,
      "assin2_sts": 73.69,
      "faquad_nli": 86.32,
      "hatebr_offensive": 89.65,
      "portuguese_hate_speech": 70.59,
      "tweetsentbr": 73.52
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7912,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2.5-VL-72B-Instruct (API) [2025-04-03]' (source=external, precision=?); published average 79.12 (divided by 100). API run (external) of Qwen2.5-VL-72B-Instruct (text-only tasks). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 79.12,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 86,
      "bluex": 80.53,
      "oab_exams": 68.88,
      "assin2_rte": 94.73,
      "assin2_sts": 75.96,
      "faquad_nli": 84.47,
      "hatebr_offensive": 88.11,
      "portuguese_hate_speech": 76.96,
      "tweetsentbr": 56.45
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "nemotron-4-340b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7778,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-06-30",
    "note": "Open PT-LLM Leaderboard row 'nvidia/Nemotron-4-340B-Instruct (Nvidia API) [2024-06-30]' (source=external, precision=?); published average 77.78 (divided by 100). API run on the NVIDIA API (external). Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 77.78,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 66.48,
      "bluex": 65.79,
      "oab_exams": 70.21,
      "assin2_rte": 94.89,
      "assin2_sts": 78.58,
      "faquad_nli": 81.94,
      "hatebr_offensive": 86.42,
      "portuguese_hate_speech": 77.62,
      "tweetsentbr": 78.09
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "open-pt-llm-avg9",
    "score": 0.7688,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-12-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen/QwQ-32B-Preview' (source=hub, precision=bfloat16); published average 76.88 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.88,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 83.41,
      "bluex": 76.63,
      "oab_exams": 63.51,
      "assin2_rte": 92.08,
      "assin2_sts": 83.08,
      "faquad_nli": 80.61,
      "hatebr_offensive": 92.07,
      "portuguese_hate_speech": 55.09,
      "tweetsentbr": 65.45
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7681,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-30B-A3B' (source=hub, precision=bfloat16); published average 76.81 (divided by 100). Local harness run, bfloat16. April 2025 hybrid checkpoint; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.81,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 87.4,
      "bluex": 83.03,
      "oab_exams": 31.16,
      "assin2_rte": 92.92,
      "assin2_sts": 78.22,
      "faquad_nli": 84.85,
      "hatebr_offensive": 86.62,
      "portuguese_hate_speech": 73.23,
      "tweetsentbr": 73.82
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "open-pt-llm-avg9",
    "score": 0.7678,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-25",
    "note": "Open PT-LLM Leaderboard row 'GPT 4o Mini (2024-07-18) [2024-07-25]' (source=external, precision=?); published average 76.78 (divided by 100). API run (external); dated snapshot gpt-4o-mini-2024-07-18. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.78,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 76.7,
      "bluex": 68.43,
      "oab_exams": 60.14,
      "assin2_rte": 94.28,
      "assin2_sts": 72.59,
      "faquad_nli": 81.98,
      "hatebr_offensive": 86.82,
      "portuguese_hate_speech": 75.01,
      "tweetsentbr": 75.09
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen-turbo-2024-11-01",
    "bench": "open-pt-llm-avg9",
    "score": 0.7649,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Qwen-Turbo (2024-11-01) [2025-04-03]' (source=external, precision=?); published average 76.49 (divided by 100). API run (external); dated snapshot qwen-turbo-2024-11-01. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.49,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 77.96,
      "bluex": 70.79,
      "oab_exams": 60.91,
      "assin2_rte": 92.6,
      "assin2_sts": 76.4,
      "faquad_nli": 81.28,
      "hatebr_offensive": 85.68,
      "portuguese_hate_speech": 72.39,
      "tweetsentbr": 70.38
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen3-4b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7618,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-4B' (source=hub, precision=bfloat16); published average 76.18 (divided by 100). Local harness run, bfloat16. April 2025 hybrid checkpoint; thinking setting not stated. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 76.18,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 84.11,
      "bluex": 78.03,
      "oab_exams": 47.97,
      "assin2_rte": 92.93,
      "assin2_sts": 70.22,
      "faquad_nli": 82.52,
      "hatebr_offensive": 87.57,
      "portuguese_hate_speech": 72.33,
      "tweetsentbr": 69.91
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemini-1-5-flash-8b-001",
    "bench": "open-pt-llm-avg9",
    "score": 0.7543,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Gemini 1.5 Flash 8B (001) [2025-04-03]' (source=external, precision=?); published average 75.43 (divided by 100). API run (external); versioned gemini-1.5-flash-8b-001. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 75.43,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 76.42,
      "bluex": 64.67,
      "oab_exams": 56.04,
      "assin2_rte": 93.29,
      "assin2_sts": 76.39,
      "faquad_nli": 79.37,
      "hatebr_offensive": 85.05,
      "portuguese_hate_speech": 73.91,
      "tweetsentbr": 73.77
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "open-pt-llm-avg9",
    "score": 0.7415,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-13",
    "note": "Open PT-LLM Leaderboard row 'Claude-3 Haiku (20240307) [2024-04-13]' (source=external, precision=?); published average 74.15 (divided by 100). API run (external); dated snapshot claude-3-haiku-20240307. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 74.15,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 77.19,
      "bluex": 66.62,
      "oab_exams": 62.69,
      "assin2_rte": 91.84,
      "assin2_sts": 78.92,
      "faquad_nli": 63.41,
      "hatebr_offensive": 80.24,
      "portuguese_hate_speech": 73.42,
      "tweetsentbr": 73.03
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemma-2-9b-simpo",
    "bench": "open-pt-llm-avg9",
    "score": 0.7328,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-10",
    "note": "Open PT-LLM Leaderboard row 'princeton-nlp/gemma-2-9b-it-SimPO' (source=hub, precision=bfloat16); published average 73.28 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 73.28,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 75.09,
      "bluex": 65.37,
      "oab_exams": 54.21,
      "assin2_rte": 93.82,
      "assin2_sts": 77.82,
      "faquad_nli": 70.45,
      "hatebr_offensive": 89.76,
      "portuguese_hate_speech": 66.68,
      "tweetsentbr": 66.28
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "open-pt-llm-avg9",
    "score": 0.7223,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-03-08",
    "note": "Open PT-LLM Leaderboard row 'GPT-3.5 Turbo (0125) [2024-03-08]' (source=external, precision=?); published average 72.23 (divided by 100). API run (external); dated snapshot gpt-3.5-turbo-0125. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 72.23,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 72.15,
      "bluex": 62.45,
      "oab_exams": 54.31,
      "assin2_rte": 88.23,
      "assin2_sts": 73.78,
      "faquad_nli": 74.64,
      "hatebr_offensive": 80.56,
      "portuguese_hate_speech": 73.64,
      "tweetsentbr": 70.29
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "open-pt-llm-avg9",
    "score": 0.7109,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-22",
    "note": "Open PT-LLM Leaderboard row 'WizardLMTeam/WizardLM-70B-V1.0' (source=hub, precision=float16); published average 71.09 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 71.09,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 68.23,
      "bluex": 59.81,
      "oab_exams": 45.74,
      "assin2_rte": 91.83,
      "assin2_sts": 82.37,
      "faquad_nli": 75.91,
      "hatebr_offensive": 85.64,
      "portuguese_hate_speech": 69.79,
      "tweetsentbr": 60.53
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "open-pt-llm-avg9",
    "score": 0.7095,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-12-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-3.2-11B-Vision-Instruct' (source=hub, precision=bfloat16); published average 70.95 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 70.95,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 70.82,
      "bluex": 59.25,
      "oab_exams": 52.16,
      "assin2_rte": 92.52,
      "assin2_sts": 76.11,
      "faquad_nli": 74.9,
      "hatebr_offensive": 86.11,
      "portuguese_hate_speech": 64.04,
      "tweetsentbr": 62.68
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "open-pt-llm-avg9",
    "score": 0.653,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-03-05",
    "note": "Open PT-LLM Leaderboard row 'lmsys/vicuna-33b-v1.3' (source=hub, precision=float16); published average 65.3 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 65.3,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 62.28,
      "bluex": 50.21,
      "oab_exams": 42.46,
      "assin2_rte": 82.21,
      "assin2_sts": 74.05,
      "faquad_nli": 72.65,
      "hatebr_offensive": 75.79,
      "portuguese_hate_speech": 68.14,
      "tweetsentbr": 59.87
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "vicuna-13b-v1-5",
    "bench": "open-pt-llm-avg9",
    "score": 0.6367,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'lmsys/vicuna-13b-v1.5' (source=hub, precision=float16); published average 63.67 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 63.67,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 55.77,
      "bluex": 44.23,
      "oab_exams": 39.95,
      "assin2_rte": 87.05,
      "assin2_sts": 73.73,
      "faquad_nli": 70.42,
      "hatebr_offensive": 76.02,
      "portuguese_hate_speech": 68.06,
      "tweetsentbr": 57.82
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "open-pt-llm-avg9",
    "score": 0.5703,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'lmsys/vicuna-7b-v1.5' (source=hub, precision=float16); published average 57.03 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 57.03,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 50.59,
      "bluex": 41.31,
      "oab_exams": 36.08,
      "assin2_rte": 79.57,
      "assin2_sts": 50.76,
      "faquad_nli": 59.38,
      "hatebr_offensive": 69.81,
      "portuguese_hate_speech": 65.87,
      "tweetsentbr": 59.89
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5006,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3-8B-Instruct' (source=hub, precision=bfloat16); published average 50.06 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "raw_value": 50.06,
    "scale_divisor": 100,
    "components": {
      "enem_challenge": 2.1,
      "bluex": 0.14,
      "oab_exams": 0.09,
      "assin2_rte": 90.91,
      "assin2_sts": 75.17,
      "faquad_nli": 75.16,
      "hatebr_offensive": 86.99,
      "portuguese_hate_speech": 59.82,
      "tweetsentbr": 60.16
    },
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-mmlu",
    "score": 0.869,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'MMLU' = 86.9; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 86.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.772,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'GPQA' = 77.2; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 77.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-math500",
    "score": 0.979,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'MATH' = 97.9; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 97.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-humaneval",
    "score": 0.976,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-01",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'HumanEval' = 97.6; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. Row added 2025-02-01 (value first published in commit 9c45dc14f3).",
    "raw_value": 97.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-mgsm",
    "score": 0.92,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'MGSM' = 92.0; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 92,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-drop-f1",
    "score": 0.806,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'DROP (F1, 3-shot)' = 80.6; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 80.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o3-mini",
    "bench": "simple-evals-simpleqa",
    "score": 0.138,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o3-mini-high', column 'SimpleQA' = 13.8; prompt 'n/a'. Reasoning effort high (highest listed row; footnote 9: default is medium). Only o3-mini snapshot is o3-mini-2025-01-31. Row added 2025-01-31 (value first published in commit 5ca150b271).",
    "raw_value": 13.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-mmlu",
    "score": 0.918,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'MMLU' = 91.8; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 91.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.757,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'GPQA' = 75.7; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 75.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-math500",
    "score": 0.964,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'MATH' = 96.4; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 96.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-mgsm",
    "score": 0.893,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'MGSM' = 89.3; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 89.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-drop-f1",
    "score": 0.902,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'DROP (F1, 3-shot)' = 90.2; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 90.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "simple-evals-simpleqa",
    "score": 0.426,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-12-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1', column 'SimpleQA' = 42.6; prompt 'n/a'. Unversioned row 'o1' added in commit 45a56a6 'O1 evals' on 2024-12-18, the day after release; OpenAI has published only one o1 API snapshot (o1-2024-12-17). Default reasoning effort (no effort variants listed). Row added 2024-12-18 (value first published in commit 45a56a6a0b).",
    "raw_value": 42.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-mmlu",
    "score": 0.908,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'MMLU' = 90.8; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 90.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.733,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'GPQA' = 73.3; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 73.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-math500",
    "score": 0.855,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'MATH' = 85.5; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 85.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-humaneval",
    "score": 0.924,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'HumanEval' = 92.4; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 92.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-mgsm",
    "score": 0.908,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'MGSM' = 90.8; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 90.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-drop-f1",
    "score": 0.748,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'DROP (F1, 3-shot)' = 74.8; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 74.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-preview",
    "bench": "simple-evals-simpleqa",
    "score": 0.424,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-preview', column 'SimpleQA' = 42.4; prompt 'n/a'. Only o1-preview snapshot (o1-preview-2024-09-12). Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 42.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-mmlu",
    "score": 0.852,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'MMLU' = 85.2; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 85.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.6,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'GPQA' = 60.0; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 60,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-math500",
    "score": 0.9,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'MATH' = 90.0; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 90,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-humaneval",
    "score": 0.924,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'HumanEval' = 92.4; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 92.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-mgsm",
    "score": 0.899,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'MGSM' = 89.9; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 89.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-drop-f1",
    "score": 0.839,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'DROP (F1, 3-shot)' = 83.9; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 83.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "o1-mini",
    "bench": "simple-evals-simpleqa",
    "score": 0.076,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'o1-mini', column 'SimpleQA' = 07.6; prompt 'n/a'. Only o1-mini snapshot (o1-mini-2024-09-12). Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 7.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-mmlu",
    "score": 0.801,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'MMLU' = 80.1; prompt 'assistant'. Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 80.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.503,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'GPQA' = 50.3; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 50.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-math500",
    "score": 0.623,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'MATH' = 62.3; prompt 'assistant'. MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 62.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-humaneval",
    "score": 0.87,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'HumanEval' = 87.0; prompt 'assistant'. Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 87,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-mgsm",
    "score": 0.73,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'MGSM' = 73.0; prompt 'assistant'. Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 73,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-drop-f1",
    "score": 0.822,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'DROP (F1, 3-shot)' = 82.2; prompt 'assistant'. Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 82.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "simple-evals-simpleqa",
    "score": 0.076,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-04-10",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.1-nano-2025-04-14', column 'SimpleQA' = 07.6; prompt 'assistant'. Row added 2025-04-10 (value first published in commit 7c68bb2cfe).",
    "raw_value": 7.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-mmlu",
    "score": 0.857,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'MMLU' = 85.7; prompt 'assistant'. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 85.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.46,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'GPQA' = 46.0; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 46,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-humaneval",
    "score": 0.902,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'HumanEval' = 90.2; prompt 'assistant'. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 90.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-mgsm",
    "score": 0.903,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'MGSM' = 90.3; prompt 'assistant'. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 90.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-drop-f1",
    "score": 0.815,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'DROP (F1, 3-shot)' = 81.5; prompt 'assistant'. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 81.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "simple-evals-simpleqa",
    "score": 0.388,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-11-21",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-11-20', column 'SimpleQA' = 38.8; prompt 'assistant'. Row added 2024-11-21 (value first published in commit 6a326d390d).",
    "raw_value": 38.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-mmlu",
    "score": 0.887,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'MMLU' = 88.7; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 88.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.531,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'GPQA' = 53.1; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 53.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-math",
    "score": 0.759,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'MATH' = 75.9; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 75.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-humaneval",
    "score": 0.902,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'HumanEval' = 90.2; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 90.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-mgsm",
    "score": 0.9,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'MGSM' = 90.0; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 90,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-drop-f1",
    "score": 0.798,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'DROP (F1, 3-shot)' = 79.8; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 79.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "simple-evals-simpleqa",
    "score": 0.401,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-08-06', column 'SimpleQA' = 40.1; prompt 'assistant'. Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 40.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-mmlu",
    "score": 0.872,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'MMLU' = 87.2; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 87.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.499,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'GPQA' = 49.9; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 49.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-math",
    "score": 0.766,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'MATH' = 76.6; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 76.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-humaneval",
    "score": 0.91,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'HumanEval' = 91.0; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 91,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-mgsm",
    "score": 0.899,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'MGSM' = 89.9; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 89.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-drop-f1",
    "score": 0.837,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-08-06",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'DROP (F1, 3-shot)' = 83.7; prompt 'assistant'. Row added 2024-08-06 (value first published in commit d2537227d7).",
    "raw_value": 83.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "simple-evals-simpleqa",
    "score": 0.39,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-2024-05-13', column 'SimpleQA' = 39.0; prompt 'assistant'. Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 39,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-mmlu",
    "score": 0.82,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'MMLU' = 82.0; prompt 'assistant'. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 82,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.402,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'GPQA' = 40.2; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 40.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-math",
    "score": 0.702,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'MATH' = 70.2; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 70.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-humaneval",
    "score": 0.872,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'HumanEval' = 87.2; prompt 'assistant'. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 87.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-mgsm",
    "score": 0.87,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'MGSM' = 87.0; prompt 'assistant'. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 87,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-drop-f1",
    "score": 0.797,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-07-18",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'DROP (F1, 3-shot)' = 79.7; prompt 'assistant'. Row added 2024-07-18 (value first published in commit 3c471d16c6).",
    "raw_value": 79.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-simpleqa",
    "score": 0.095,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4o-mini-2024-07-18', column 'SimpleQA' = 09.5; prompt 'assistant'. Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 9.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-mmlu",
    "score": 0.908,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'MMLU' = 90.8; prompt 'assistant'. Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 90.8,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.695,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'GPQA' = 69.5; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py) with an answer-regex tweak (footnote 8). Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 69.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-math500",
    "score": 0.871,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'MATH' = 87.1; prompt 'assistant'. MATH column is MATH-500 for models on or after o1 (footnote 6). Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 87.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-humaneval",
    "score": 0.886,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'HumanEval' = 88.6; prompt 'assistant'. Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 88.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-mgsm",
    "score": 0.869,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'MGSM' = 86.9; prompt 'assistant'. Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 86.9,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-drop-f1",
    "score": 0.834,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'DROP (F1, 3-shot)' = 83.4; prompt 'assistant'. Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 83.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "simple-evals-simpleqa",
    "score": 0.625,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2025-02-26",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4.5-preview-2025-02-27', column 'SimpleQA' = 62.5; prompt 'assistant'. Row added 2025-02-26 (value first published in commit 3fe6471649).",
    "raw_value": 62.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-mmlu",
    "score": 0.867,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'MMLU' = 86.7; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 86.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.493,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'GPQA' = 49.3; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 49.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-math",
    "score": 0.734,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'MATH' = 73.4; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 73.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-humaneval",
    "score": 0.882,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'HumanEval' = 88.2; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 88.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-mgsm",
    "score": 0.896,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'MGSM' = 89.6; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 89.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-drop-f1",
    "score": 0.86,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-04-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'DROP (F1, 3-shot)' = 86.0; prompt 'assistant'. Row added 2024-04-11 (value first published in commit fea8034e54).",
    "raw_value": 86,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "simple-evals-simpleqa",
    "score": 0.242,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-10-30",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-turbo-2024-04-09', column 'SimpleQA' = 24.2; prompt 'assistant'. Row added 2024-10-30 (value first published in commit f933bc2291).",
    "raw_value": 24.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-mmlu",
    "score": 0.854,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'MMLU' = 85.4; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 85.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.414,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'GPQA' = 41.4; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 41.4,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-math",
    "score": 0.645,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'MATH' = 64.5; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 64.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-humaneval",
    "score": 0.866,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'HumanEval' = 86.6; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 86.6,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-mgsm",
    "score": 0.851,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-05-13",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'MGSM' = 85.1; prompt 'assistant'. Row added 2024-05-13 (value first published in commit 294cb1fb18).",
    "raw_value": 85.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "simple-evals-drop-f1",
    "score": 0.815,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-04-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-0125-preview', column 'DROP (F1, 3-shot)' = 81.5; prompt 'assistant'. Row added 2024-04-11 (value first published in commit fea8034e54).",
    "raw_value": 81.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-mmlu",
    "score": 0.847,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'MMLU' = 84.7; prompt 'assistant'. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 84.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.425,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'GPQA' = 42.5; prompt 'assistant'. GPQA uses the simple-evals default variant 'diamond' (gpqa_eval.py default since the initial commit); this value was first published before the answer-regex tweak note (footnote 8) appeared on 2025-01-31 and is unchanged since, so it was scored with the earlier answer regex. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 42.5,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-math",
    "score": 0.643,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'MATH' = 64.3; prompt 'assistant'. MATH column for pre-o1 models: the value was first published before simple-evals adopted MATH-500 (footnote 6 applies to o1 and later) and is unchanged since, so it is the original MATH test set, not MATH-500. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 64.3,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-humaneval",
    "score": 0.837,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'HumanEval' = 83.7; prompt 'assistant'. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 83.7,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-mgsm",
    "score": 0.871,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'MGSM' = 87.1; prompt 'assistant'. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 87.1,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "simple-evals-drop-f1",
    "score": 0.832,
    "source": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md",
    "by": "lab",
    "date": "2024-09-11",
    "note": "openai/simple-evals README 'Benchmark Results' table (OpenAI's own evaluation, zero-shot CoT); row 'gpt-4-1106-preview', column 'DROP (F1, 3-shot)' = 83.2; prompt 'assistant'. Row added 2024-09-11 (value first published in commit fb3f1efd0f).",
    "raw_value": 83.2,
    "scale_divisor": 100,
    "source_id": "openai-simple-evals-readme"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "robloxqa-v2-mc",
    "score": 0.923333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2770/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Claude Sonnet 4 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files anthropic-claude-sonnet-4_mc_effort-medium.json5 / anthropic-claude-sonnet-4_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "luau-leetcode-full-pass",
    "score": 0.682692,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 40+68+34=142/208. Roblox LLM Leaderboard entry 'Claude Sonnet 4 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files anthropic-claude-sonnet-4_mc_effort-medium.json5 / anthropic-claude-sonnet-4_effort-medium.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "robloxqa-v2-mc",
    "score": 0.905667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "2717/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Deepseek V4 Flash' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash); no reasoning setting listed; run files deepseek-deepseek-v4-flash_mc.json5 / deepseek-deepseek-v4-flash.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "luau-leetcode-full-pass",
    "score": 0.572115,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/3fcd29933bf7276ce5470d9de8c235fe5684fc43/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-07",
    "note": "fullPass 41+58+20=119/208. Roblox LLM Leaderboard entry 'Deepseek V4 Flash' (https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash); no reasoning setting listed; run files deepseek-deepseek-v4-flash_mc.json5 / deepseek-deepseek-v4-flash.json5. Date = leaderboard generatedAt 2026-09-07. Space revision 3fcd29933bf7276ce5470d9de8c235fe5684fc43.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "scicode-main-official",
    "score": 0.092,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2025-02-01",
    "note": "SciCode repository README leaderboard; row 'OpenAI o3-mini-high'; Main Problem Resolve Rate 9.2% (subproblem 34.4% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; results for OpenAI o3-mini added 2025-02-01 per README news. Reasoning effort high (highest listed; the low and medium rows are not imported). Only o3-mini snapshot is o3-mini-2025-01-31.",
    "raw_value": 9.2,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "o1-preview",
    "bench": "scicode-main-official",
    "score": 0.077,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-10-01",
    "note": "SciCode repository README leaderboard; row 'OpenAI o1-preview'; Main Problem Resolve Rate 7.7% (subproblem 28.5% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; o1-mini and o1-preview results added 2024-10-01 per README news. Only o1-preview snapshot (o1-preview-2024-09-12).",
    "raw_value": 7.7,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "scicode-main-official",
    "score": 0.046,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Claude3.5-Sonnet'; Main Problem Resolve Rate 4.6% (subproblem 26.0% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; evaluated before 2024-07-18, when claude-3-5-sonnet-20240620 was the only Claude 3.5 Sonnet snapshot (the README separately lists the Oct-2024 '(new)' model).",
    "raw_value": 4.6,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "scicode-main-official",
    "score": 0.046,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-11-04",
    "note": "SciCode repository README leaderboard; row 'Claude3.5-Sonnet (new)'; Main Problem Resolve Rate 4.6% (subproblem 25.3% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; Claude Sonnet 3.5 (new) results added 2024-11-04 per README news; 'new' is the 2024-10-22 snapshot.",
    "raw_value": 4.6,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'GPT-4o'; Main Problem Resolve Rate 1.5% (subproblem 25.0% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; the paper names only 'GPT-4o', evaluated before 2024-07-18, when gpt-4o-2024-05-13 was the only GPT-4o snapshot (gpt-4o-2024-08-06 was released on 2024-08-06).",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'GPT-4-Turbo'; Main Problem Resolve Rate 1.5% (subproblem 22.9% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; the paper states the 'gpt-4-turbo-2024-04-09' snapshot.",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-10-01",
    "note": "SciCode repository README leaderboard; row 'OpenAI o1-mini'; Main Problem Resolve Rate 1.5% (subproblem 22.2% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; o1-mini and o1-preview results added 2024-10-01 per README news. Only o1-mini snapshot (o1-mini-2024-09-12).",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Claude3-Opus'; Main Problem Resolve Rate 1.5% (subproblem 21.5% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; claude-3-opus-20240229 is the only Claude 3 Opus snapshot.",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Claude3-Sonnet'; Main Problem Resolve Rate 1.5% (subproblem 17.0% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; claude-3-sonnet-20240229 is the only Claude 3 Sonnet snapshot.",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "qwen2-72b",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Qwen2-72B-Instruct'; Main Problem Resolve Rate 1.5% (subproblem 17.0% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; exact open checkpoint Qwen2-72B-Instruct.",
    "raw_value": 1.5,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "scicode-main-official",
    "score": 0,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Mixtral-8x22B-Instruct'; Main Problem Resolve Rate 0.0% (subproblem 16.3% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values; Mixtral-8x22B-Instruct-v0.1 is the only released 8x22B instruct checkpoint.",
    "raw_value": 0,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "llama-3-70b",
    "bench": "scicode-main-official",
    "score": 0,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Llama-3-70B-Chat'; Main Problem Resolve Rate 0.0% (subproblem 14.6% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values (the paper's model list calls it Llama-3-70B-Instruct).",
    "raw_value": 0,
    "scale_divisor": 100,
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.566,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o-mini/miniwob.json",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 56.6% +/- 2.0 (divided by 100); study 2024-10-25_06-08-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o-mini as a backend'; unversioned, but OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.174,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o-mini/webarena.json",
    "by": "independent",
    "date": "2024-11-29",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 17.4% +/- 1.3 (divided by 100); study c6bdeb87-9879-4c06-aa70-00d895001156; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses GPT-4o-mini as a backend'; unversioned, but OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.27,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o-mini/workarena-l1.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 27% +/- 2.4 (divided by 100); study 2024-10-23_14-17-40; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o-mini as a backend'; unversioned, but OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.013,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o-mini/workarena-l2.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 1.3% +/- 0.7 (divided by 100); study 2024-10-23_17-10-46; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o-mini as a backend'; unversioned, but OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.678,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-o1-mini/miniwob.json",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-o1-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 67.8% +/- 1.9 (divided by 100); study 2024-10-25_06-08-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses o1-mini as a backend'; o1-mini-2024-09-12 is the only o1-mini snapshot. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.286,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-o1-mini/webarena.json",
    "by": "independent",
    "date": "2024-11-30",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-o1-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 28.6% +/- 1.6 (divided by 100); study 1827983d-5e84-4b63-ad49-bf45ec2a6348; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses o1-mini as a backend'; o1-mini-2024-09-12 is the only o1-mini snapshot. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.567,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-o1-mini/workarena-l1.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-o1-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 56.7% +/- 2.7 (divided by 100); study 2024-10-23_14-17-40; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses o1-mini as a backend'; o1-mini-2024-09-12 is the only o1-mini snapshot. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.149,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-o1-mini/workarena-l2.json",
    "by": "independent",
    "date": "2024-10-24",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-o1-mini' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 14.9% +/- 2.3 (divided by 100); study 2024-10-23_17-10-46; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses o1-mini as a backend'; o1-mini-2024-09-12 is the only o1-mini snapshot. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.446,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-3.7-Sonnet/webarena.json",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-3.7-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 44.6% +/- 2.5 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README names claude-3-7-sonnet-20250219. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Thinking not stated. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.707,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-4-Sonnet/miniwob.json",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-4-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 70.7% +/- 1.8 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README names claude-sonnet-4-20250514. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Thinking not stated. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.633,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-4-Sonnet/workarena-l1.json",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-4-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 63.3% +/- 2.7 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README names claude-sonnet-4-20250514. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Thinking not stated. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.404,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-4-Sonnet/workarena-l2.json",
    "by": "independent",
    "date": "2025-08-07",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-4-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 40.4% +/- 3.2 (divided by 100); study 2025-08-07_21-09-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README names claude-sonnet-4-20250514. The date_time 2025-08-07 21:09:16 is shared by every 2025 GenericAgent entry and may be a placeholder. Thinking not stated. Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "cptub-speakleash-avg",
    "score": 0.786842,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Large-Instruct-2407'; Space Average = mean(Sentiment 4.231, Language understanding 4.000, Phraseology 3.860, Tricky questions 3.646) = 3.9342 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.934209,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "cptub-speakleash-avg",
    "score": 0.739815,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'alpindale/WizardLM-2-8x22B (API)'; Space Average = mean(Sentiment 3.705, Language understanding 3.815, Phraseology 4.220, Tricky questions 3.056) = 3.6991 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. alpindale/WizardLM-2-8x22B re-uploads Microsoft's withdrawn WizardLM-2-8x22B weights.",
    "raw_value": 3.699077,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "cptub-speakleash-avg",
    "score": 0.71215,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mixtral-8x22B-Instruct-v0.1 (API)'; Space Average = mean(Sentiment 3.782, Language understanding 3.675, Phraseology 3.550, Tricky questions 3.236) = 3.5608 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.560752,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "cptub-speakleash-avg",
    "score": 0.8584,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'gemini-2.0-flash-001'; Space Average = mean(Sentiment 4.519, Language understanding 4.320, Phraseology 4.340, Tricky questions 3.989) = 4.2920 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 4.291999,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "cptub-speakleash-avg",
    "score": 0.779548,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'mistralai/Mistral-Small-3.1-24B-Instruct-2503 (API FP8)'; Space Average = mean(Sentiment 4.135, Language understanding 3.885, Phraseology 4.150, Tricky questions 3.421) = 3.8977 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.897741,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "kimi-k2-0905",
    "bench": "cptub-speakleash-avg",
    "score": 0.79668,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'moonshotai/Kimi-K2-Instruct-0905 (API)'; Space Average = mean(Sentiment 4.391, Language understanding 4.180, Phraseology 3.430, Tricky questions 3.933) = 3.9834 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "raw_value": 3.983402,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "cptub-speakleash-avg",
    "score": 0.707395,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Qwen/Qwen3-30B-A3B non-thinking (API)'; Space Average = mean(Sentiment 3.718, Language understanding 3.390, Phraseology 3.495, Tricky questions 3.545) = 3.5370 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Original April 2025 hybrid Qwen3-30B-A3B run with thinking disabled (the only published setting for it).",
    "raw_value": 3.536973,
    "scale_divisor": 5,
    "source_id": "speakleash-cptub"
  },
  {
    "model": "phi-3-mini-4k-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.2805,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'microsoft/Phi-3-mini-4k-instruct', model path microsoft/Phi-3-mini-4k-instruct; eq-bench_v2_pl score 28.05 (divided by 100); 159.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 13:34:56. Run completed 2024-06-19, before Microsoft's July 2024 update, so the April 2024 checkpoint.",
    "raw_value": 28.05,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "command-r-03-2024",
    "bench": "polish-eq-bench-v2",
    "score": 0.5643,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'CohereForAI/c4ai-command-r-v01', model path CohereForAI/c4ai-command-r-v01; eq-bench_v2_pl score 56.43 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 14:14:54.",
    "raw_value": 56.43,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "polish-eq-bench-v2",
    "score": 0.6956,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-27",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'alpindale/WizardLM-2-8x22B', model path alpindale/WizardLM-2-8x22B; eq-bench_v2_pl score 69.56 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-27 01:50:42. alpindale/WizardLM-2-8x22B is a byte-identical re-upload of Microsoft's withdrawn WizardLM-2-8x22B weights.",
    "raw_value": 69.56,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "mistral-large-2407",
    "bench": "polish-eq-bench-v2",
    "score": 0.7807,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-08-24",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'mistralai/Mistral-Large-Instruct-2407', model path mistralai/Mistral-Large-Instruct-2407; eq-bench_v2_pl score 78.07 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-08-24 21:51:53.",
    "raw_value": 78.07,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "polish-eq-bench-v2",
    "score": 0.7723,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-08-25",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model path meta-llama/Meta-Llama-3.1-405B-Instruct-FP8; eq-bench_v2_pl score 77.23 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2024-08-25 20:59:04. Source checkpoint is specifically the FP8 release.",
    "raw_value": 77.23,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "polish-eq-bench-v2",
    "score": 0.7115,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-08-25",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'gpt-4o-mini-2024-07-18', model path gpt-4o-mini-2024-07-18; eq-bench_v2_pl score 71.15 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2024-08-25 21:17:34.",
    "raw_value": 71.15,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "polish-eq-bench-v2",
    "score": 0.7515,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-08-25",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'gpt-4o-2024-08-06', model path gpt-4o-2024-08-06; eq-bench_v2_pl score 75.15 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2024-08-25 21:24:35.",
    "raw_value": 75.15,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "polish-eq-bench-v2",
    "score": 0.7777,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-08-25",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'gpt-4-turbo-2024-04-09', model path gpt-4-turbo-2024-04-09; eq-bench_v2_pl score 77.77 (divided by 100); 164.0 of 171 questions parseable; 1 iteration(s); inference engine openai; completed 2024-08-25 21:31:42.",
    "raw_value": 77.77,
    "scale_divisor": 100,
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "mistral-large-2407",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.473485,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'Mistral-Large-Instruct-2407', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T04-22-52-04-00_gpqa-diamond_SuZUZxGdqS2ZecbLRNkKd4.eval). mistralai/Mistral-Large-Instruct-2407, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.694232,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'Mistral-Large-Instruct-2407', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T09-41-25-04-00_mmlu-pro_fyYT4aabPesfY5TpzFMPnd.eval). mistralai/Mistral-Large-Instruct-2407, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "vector-inspect-math",
    "score": 0.6574,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'Mistral-Large-Instruct-2407', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T06-33-09-04-00_math_2CmjBedAfUxqvmcHRdBgyB.eval). mistralai/Mistral-Large-Instruct-2407, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "vector-inspect-humaneval",
    "score": 0.865854,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'Mistral-Large-Instruct-2407', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2024-10-31T06-29-24-04-00_humaneval_nu8SUSGekKJWB8HLKDigYK.eval). mistralai/Mistral-Large-Instruct-2407, dtype torch.float16 per config. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.319444,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-29",
    "note": "Vector Institute Evaluation Leaderboard entry 'Command R+', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.eval). CohereForAI/c4ai-command-r-plus (the April 2024 weights; the 08-2024 release is a separate repo), config lists no dtype. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.441157,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'Command R+', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.eval). CohereForAI/c4ai-command-r-plus (the April 2024 weights; the 08-2024 release is a separate repo), config lists no dtype. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "vector-inspect-math",
    "score": 0.2626,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'Command R+', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.eval). CohereForAI/c4ai-command-r-plus (the April 2024 weights; the 08-2024 release is a separate repo), config lists no dtype. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "vector-inspect-humaneval",
    "score": 0.621951,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'Command R+', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.eval). CohereForAI/c4ai-command-r-plus (the April 2024 weights; the 08-2024 release is a separate repo), config lists no dtype. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.609848,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-15",
    "note": "Vector Institute Evaluation Leaderboard entry 'claude-3-5-sonnet-20241022', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-15T13-56-36-05-00_gpqa-diamond_eg4gFaMRENjnnYvQNtSB59.eval). Entry key is the dated API snapshot claude-3-5-sonnet-20241022. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.776263,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'claude-3-5-sonnet-20241022', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-16T19-01-05-05-00_mmlu-pro_3vi84or97gQupuj5sT6vgZ.eval). Entry key is the dated API snapshot claude-3-5-sonnet-20241022. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "vector-inspect-math",
    "score": 0.7942,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'claude-3-5-sonnet-20241022', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-16T12-29-54-05-00_math_NvNQU58M8r3fpiwPGnvq8h.eval). Entry key is the dated API snapshot claude-3-5-sonnet-20241022. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "vector-inspect-humaneval",
    "score": 0.945122,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-16",
    "note": "Vector Institute Evaluation Leaderboard entry 'claude-3-5-sonnet-20241022', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2025-01-16T11-26-12-05-00_humaneval_kUASiaNd9uZfWvCwYHhdF5.eval). Entry key is the dated API snapshot claude-3-5-sonnet-20241022. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.383838,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o-mini', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.eval). Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.639628,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o-mini', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.eval). Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vector-inspect-math",
    "score": 0.633,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o-mini', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.eval). Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vector-inspect-humaneval",
    "score": 0.859756,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-30",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o-mini', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.eval). Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.755051,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-17",
    "note": "Vector Institute Evaluation Leaderboard entry 'o1', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-17T11-53-53-05-00_gpqa-diamond_EJV7ULFSQLRoFTEqsv3t6q.eval). Unversioned alias 'o1'; o1-2024-12-17 is the only o1 snapshot (o1-preview is a separate model). Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.844747,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-20",
    "note": "Vector Institute Evaluation Leaderboard entry 'o1', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-20T14-02-37-05-00_mmlu-pro_EvDzvqaahQwhv6fJovN4BT.eval). Unversioned alias 'o1'; o1-2024-12-17 is the only o1 snapshot (o1-preview is a separate model). Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "vector-inspect-math",
    "score": 0.959,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-17",
    "note": "Vector Institute Evaluation Leaderboard entry 'o1', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2025-01-17T15-03-22-05-00_math_6BbvHFF8hLMsVYozyNLbyQ.eval). Unversioned alias 'o1'; o1-2024-12-17 is the only o1 snapshot (o1-preview is a separate model). Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "vector-inspect-humaneval",
    "score": 0.969512,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-01-17",
    "note": "Vector Institute Evaluation Leaderboard entry 'o1', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2025-01-17T14-59-12-05-00_humaneval_RRL8GMy9NakTxUHsDVWNng.eval). Unversioned alias 'o1'; o1-2024-12-17 is the only o1 snapshot (o1-preview is a separate model). Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.736532,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Vector Institute Evaluation Leaderboard entry 'o3-mini', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-06T17-57-54-05-00_gpqa-diamond_2znyMtdc7X4LJufxXeXA8Z.eval). Unversioned alias; o3-mini-2025-01-31 is the only o3-mini snapshot. Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.792461,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Vector Institute Evaluation Leaderboard entry 'o3-mini', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-06T19-49-27-05-00_mmlu-pro_jz9woKfdKt8VMzqNFsy7kY.eval). Unversioned alias; o3-mini-2025-01-31 is the only o3-mini snapshot. Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "vector-inspect-math",
    "score": 0.969132,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Vector Institute Evaluation Leaderboard entry 'o3-mini', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2025-02-06T18-33-30-05-00_math_86Gx8n4BxhpyfaSHmRcCUm.eval). Unversioned alias; o3-mini-2025-01-31 is the only o3-mini snapshot. Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "vector-inspect-humaneval",
    "score": 0.981707,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2025-02-06",
    "note": "Vector Institute Evaluation Leaderboard entry 'o3-mini', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2025-02-06T20-58-48-05-00_humaneval_Dkod7CS9RmbbogYx9aEXtx.eval). Unversioned alias; o3-mini-2025-01-31 is the only o3-mini snapshot. Reasoning effort not published. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "webapp1k-react-pass1",
    "score": 0.8808,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-10-24",
    "note": "Entry 'claude-3.5-sonnet-20240620' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "webapp1k-react-pass1",
    "score": 0.7002,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'deepseek-coder-v2-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-07-22, before the 0724 update, so the original DeepSeek-Coder-V2-Instruct (0614). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "webapp1k-react-pass1",
    "score": 0.4606,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'deepseek-coder-v2-lite-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. DeepSeek-Coder-V2-Lite-Instruct (16B MoE). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "webapp1k-react-pass1",
    "score": 0.8338,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-09-14",
    "note": "Entry 'deepseek-v2.5' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-09-14 with an unchanged value since, after DeepSeek-V2.5 (2024-09-05) and before V2.5-1210. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "webapp1k-react-pass1",
    "score": 0.8702,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'gpt-4o-2024-05-13' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "webapp1k-react-pass1",
    "score": 0.885,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'gpt-4o-2024-08-06' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "webapp1k-react-pass1",
    "score": 0.8271,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'gpt-4o-mini' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one GPT-4o mini snapshot (gpt-4o-mini-2024-07-18). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "webapp1k-react-pass1",
    "score": 0.224,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-12-28",
    "note": "Entry 'llama-3.1-nemotron-70b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. NVIDIA Llama-3.1-Nemotron-70B-Instruct. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "llama-3-70b",
    "bench": "webapp1k-react-pass1",
    "score": 0.3323,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'llama-v3-70b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Fireworks-style id for Meta-Llama-3-70B-Instruct. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "llama-3-8b",
    "bench": "webapp1k-react-pass1",
    "score": 0.0679,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'llama-v3-8b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Fireworks-style id for Meta-Llama-3-8B-Instruct. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "webapp1k-react-pass1",
    "score": 0.7804,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-08-17",
    "note": "Entry 'mistral-large-2' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-08-17 with an unchanged value since, when 'Mistral Large 2' was only the 2407 release (2411 came in November 2024). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mistral-small-3",
    "bench": "webapp1k-react-pass1",
    "score": 0.555,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2025-01-31",
    "note": "Entry 'mistral-small-3' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-01-31: Mistral Small 3 (24B, 2501). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "webapp1k-react-pass1",
    "score": 0.3074,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'mixtral-8x22b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Mistral released one Mixtral 8x22B instruct checkpoint (v0.1). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "webapp1k-react-pass1",
    "score": 0.1269,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Entry 'mixtral-8x7b-instruct' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. Mistral released one Mixtral 8x7B instruct checkpoint (v0.1); the published value changed from 0.0 (2024-07-23) to this value in a later commit. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "webapp1k-react-pass1",
    "score": 0.939,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-09-14",
    "note": "Entry 'o1-mini' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o1-mini snapshot (o1-mini-2024-09-12). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o1-preview",
    "bench": "webapp1k-react-pass1",
    "score": 0.952,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-09-14",
    "note": "Entry 'o1-preview' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o1-preview snapshot (o1-preview-2024-09-12). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "webapp1k-react-pass1",
    "score": 0.961,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2025-02-01",
    "note": "Entry 'o3-mini' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o3-mini snapshot (o3-mini-2025-01-31). Reasoning effort not stated. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "qwen-max-2025-01-25",
    "bench": "webapp1k-react-pass1",
    "score": 0.576,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2025-01-28",
    "note": "Entry 'qwen2p5-max' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-01-28; Qwen2.5-Max's only API snapshot then was qwen-max-2025-01-25. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "qwq-32b",
    "bench": "webapp1k-react-pass1",
    "score": 0.87,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2025-03-08",
    "note": "Entry 'qwq-32b' in results.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-03-08, after the final QwQ-32B release (2025-03-05); not QwQ-32B-Preview. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.68,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-10-24",
    "note": "Entry 'claude-3-5-sonnet-20240620' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (3000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.75,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-10-24",
    "note": "Entry 'claude-3-5-sonnet-20241022' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.666,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-02-24",
    "note": "Entry 'claude-3-7-sonnet-20250219' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.769,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-09-20",
    "note": "Entry 'claude-opus-4-1-20250805' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.763,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-09-19",
    "note": "Entry 'claude-opus-4-20250514' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.49,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-09-17",
    "note": "Entry 'deepseek-v2.5' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2024-09-17 with an unchanged value since, after DeepSeek-V2.5 (2024-09-05) and before V2.5-1210. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.641,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-10-14",
    "note": "Entry 'deepseek-v3.2-exp' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. DeepSeek V3.2-Exp. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.57,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-02",
    "note": "Entry 'gemini-2.5-flash-preview-04-17' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated preview API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.634,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-21",
    "note": "Entry 'gemini-2.5-flash-preview-05-20' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated preview API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.674,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-08",
    "note": "Entry 'gemini-2.5-pro-preview-05-06' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated preview API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.704,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-06-07",
    "note": "Entry 'gemini-2.5-pro-preview-06-05' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated preview API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.75,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-11-24",
    "note": "Entry 'gemini-3-pro-preview' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact preview API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.663,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-02-28",
    "note": "Entry 'gpt-4.5-preview-2025-02-27' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact dated API snapshot. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-5-1-codex",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.743,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-12-08",
    "note": "Entry 'gpt-5.1-codex' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. API id gpt-5.1-codex (single snapshot). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "grok-3-beta",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.602,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-04",
    "note": "Entry 'grok-3-beta' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact API id grok-3-beta. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.716,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-11-25",
    "note": "Entry 'grok-4-1-fast-reasoning' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Exact API id. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "kimi-k2-0905",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.579,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-10-26",
    "note": "Entry 'kimi-k2-instruct-0905' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Kimi K2 Instruct 0905. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mistral-large-2407",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.449,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-09-17",
    "note": "Entry 'mistral-large-2' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2024-09-17 with an unchanged value since, when 'Mistral Large 2' was only the 2407 release. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.537,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-09",
    "note": "Entry 'mistral-medium-3' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-05-09: Mistral Medium 3 (2505), before Medium 3.1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o1-mini",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.667,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-09-17",
    "note": "Entry 'o1-mini' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o1-mini snapshot (o1-mini-2024-09-12). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o1-preview",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.652,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2024-09-17",
    "note": "Entry 'o1-preview' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o1-preview snapshot (o1-preview-2024-09-12). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "o3-mini",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.696,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-02-01",
    "note": "Entry 'o3-mini' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Unversioned alias; OpenAI has published only one o3-mini snapshot (o3-mini-2025-01-31). Reasoning effort not stated. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.416,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-01-15",
    "note": "Entry 'qwen2p5-coder-32b-instruct' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Fireworks-style id for Qwen2.5-Coder-32B-Instruct. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "qwq-32b",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.616,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Entry 'qwq-32b' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-03-07, after the final QwQ-32B release; not the Preview. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "rpc-bench-text-overall",
    "score": 0.3246,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'Gemini-3-Pro' (Google, text input, status published); overall 32.46 (correctness 62.69, completeness 60.28, conciseness 52.81, f1_like 61.46); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value. Gemini 3 Pro (2025-11-18) was released only as the gemini-3-pro-preview API model; thinking level not stated.",
    "source_id": "zai-rpc-bench-leaderboard"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "rpc-bench-text-overall",
    "score": 0.2419,
    "source": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json",
    "by": "independent",
    "date": "2026-06-17",
    "note": "RPC-Bench leaderboard row 'Claude-Sonnet-4' (Anthropic, text input, status published); overall 24.19 (correctness 58.53, completeness 58.44, conciseness 41.37, f1_like 58.48); divided by 100. Reasoning effort not stated. Date is the leaderboard last_updated value. Anthropic released one Claude Sonnet 4 snapshot (claude-sonnet-4-20250514); thinking not stated.",
    "source_id": "zai-rpc-bench-leaderboard"
  },
  {
    "model": "athene-70b",
    "bench": "cruxeval-zeroeval",
    "score": 0.5062,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Athene-70B'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "athene-70b",
    "bench": "gsm8k-zeroeval",
    "score": 0.8666,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Athene-70B'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "athene-70b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7664,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Athene-70B'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "athene-70b",
    "bench": "zebralogic-zeroeval",
    "score": 0.167,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Athene-70B'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=21.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-70b",
    "bench": "cruxeval-zeroeval",
    "score": 0.5888,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-70b",
    "bench": "gsm8k-zeroeval",
    "score": 0.9303,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-70b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7801,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-70b",
    "bench": "zebralogic-zeroeval",
    "score": 0.168,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-8b",
    "bench": "cruxeval-zeroeval",
    "score": 0.3775,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-8b",
    "bench": "gsm8k-zeroeval",
    "score": 0.7847,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-8b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6166,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.97. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-8b",
    "bench": "zebralogic-zeroeval",
    "score": 0.119,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=29.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "cruxeval-zeroeval",
    "score": 0.7512,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Mistral-Large-2'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/mistral-large-2.sh at the pinned commit runs '-f mistral -m mistral-large-2407 -p Mistral-Large-2'; result files first committed 2024-07-30 (before the 2411 release).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "gsm8k-zeroeval",
    "score": 0.9553,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Mistral-Large-2'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/mistral-large-2.sh at the pinned commit runs '-f mistral -m mistral-large-2407 -p Mistral-Large-2'; result files first committed 2024-07-30 (before the 2411 release).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8297,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Mistral-Large-2'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/mistral-large-2.sh at the pinned commit runs '-f mistral -m mistral-large-2407 -p Mistral-Large-2'; result files first committed 2024-07-30 (before the 2411 release).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "zebralogic-zeroeval",
    "score": 0.29,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Mistral-Large-2'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.70. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/mistral-large-2.sh at the pinned commit runs '-f mistral -m mistral-large-2407 -p Mistral-Large-2'; result files first committed 2024-07-30 (before the 2411 release).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "cruxeval-zeroeval",
    "score": 0.4488,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Mixtral-8x7B-Instruct-v0.1'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "gsm8k-zeroeval",
    "score": 0.7013,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Mixtral-8x7B-Instruct-v0.1'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=2.27. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6317,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Mixtral-8x7B-Instruct-v0.1'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=5.51. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "zebralogic-zeroeval",
    "score": 0.087,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Mixtral-8x7B-Instruct-v0.1'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=20.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "cruxeval-zeroeval",
    "score": 0.4475,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3-mini-4k-instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.75. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Run locally from HF repo microsoft/Phi-3-mini-4k-instruct (scripts/local/phi-3.sh); ZeroEval result files committed 2024-07-21..2024-08-02, after the HF repo's 'Update model (#77)' commit 4f818b1 on 2024-07-01 (the June 2024 update), so the served weights are the June 2024 update.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "gsm8k-zeroeval",
    "score": 0.7551,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3-mini-4k-instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Run locally from HF repo microsoft/Phi-3-mini-4k-instruct (scripts/local/phi-3.sh); ZeroEval result files committed 2024-07-21..2024-08-02, after the HF repo's 'Update model (#77)' commit 4f818b1 on 2024-07-01 (the June 2024 update), so the served weights are the June 2024 update.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7034,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3-mini-4k-instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.43. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Run locally from HF repo microsoft/Phi-3-mini-4k-instruct (scripts/local/phi-3.sh); ZeroEval result files committed 2024-07-21..2024-08-02, after the HF repo's 'Update model (#77)' commit 4f818b1 on 2024-07-01 (the June 2024 update), so the served weights are the June 2024 update.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "zebralogic-zeroeval",
    "score": 0.116,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3-mini-4k-instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=59.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Run locally from HF repo microsoft/Phi-3-mini-4k-instruct (scripts/local/phi-3.sh); ZeroEval result files committed 2024-07-21..2024-08-02, after the HF repo's 'Update model (#77)' commit 4f818b1 on 2024-07-01 (the June 2024 update), so the served weights are the June 2024 update.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "cruxeval-zeroeval",
    "score": 0.5913,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-72B-Instruct'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "gsm8k-zeroeval",
    "score": 0.9265,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-72B-Instruct'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8161,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-72B-Instruct'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.29. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "zebralogic-zeroeval",
    "score": 0.214,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-72B-Instruct'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=10.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "cruxeval-zeroeval",
    "score": 0.6875,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "gsm8k-zeroeval",
    "score": 0.9447,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7963,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "zebralogic-zeroeval",
    "score": 0.187,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "cruxeval-zeroeval",
    "score": 0.8075,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "gsm8k-zeroeval",
    "score": 0.956,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.86,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.18. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "zebralogic-zeroeval",
    "score": 0.334,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "cruxeval-zeroeval",
    "score": 0.8388,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "gsm8k-zeroeval",
    "score": 0.9666,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8891,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "zebralogic-zeroeval",
    "score": 0.362,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "cruxeval-zeroeval",
    "score": 0.5475,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-haiku-20240307'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "gsm8k-zeroeval",
    "score": 0.8878,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-haiku-20240307'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7232,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-haiku-20240307'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "zebralogic-zeroeval",
    "score": 0.143,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-haiku-20240307'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "cruxeval-zeroeval",
    "score": 0.7038,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-opus-20240229'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "gsm8k-zeroeval",
    "score": 0.956,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-opus-20240229'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8254,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-opus-20240229'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.58. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "zebralogic-zeroeval",
    "score": 0.27,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-opus-20240229'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "cruxeval-zeroeval",
    "score": 0.6662,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-sonnet-20240229'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "gsm8k-zeroeval",
    "score": 0.9151,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-sonnet-20240229'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7487,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-sonnet-20240229'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.07. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "zebralogic-zeroeval",
    "score": 0.187,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-sonnet-20240229'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "gsm8k-zeroeval",
    "score": 0.9378,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0614'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Label pins the 0614 Coder-V2 release (catalogue identity note: API name DeepSeek-Coder-V2-0614).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7963,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0614'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Label pins the 0614 Coder-V2 release (catalogue identity note: API name DeepSeek-Coder-V2-0614).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "zebralogic-zeroeval",
    "score": 0.211,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0614'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=4.90. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Label pins the 0614 Coder-V2 release (catalogue identity note: API name DeepSeek-Coder-V2-0614).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "cruxeval-zeroeval",
    "score": 0.7,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2.5-0908'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/deepseek_coder.sh: '-m deepseek-chat -p deepseek-v2.5-0908'; label pins V2.5 (Sept 2024), before V2.5-1210 existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "gsm8k-zeroeval",
    "score": 0.9249,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2.5-0908'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/deepseek_coder.sh: '-m deepseek-chat -p deepseek-v2.5-0908'; label pins V2.5 (Sept 2024), before V2.5-1210 existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8035,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2.5-0908'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.22. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/deepseek_coder.sh: '-m deepseek-chat -p deepseek-v2.5-0908'; label pins V2.5 (Sept 2024), before V2.5-1210 existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "zebralogic-zeroeval",
    "score": 0.221,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2.5-0908'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=12.70. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: scripts/local/deepseek_coder.sh: '-m deepseek-chat -p deepseek-v2.5-0908'; label pins V2.5 (Sept 2024), before V2.5-1210 existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-27b",
    "bench": "cruxeval-zeroeval",
    "score": 0.5725,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-27b-it'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-27b",
    "bench": "gsm8k-zeroeval",
    "score": 0.9022,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-27b-it'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-27b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7567,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-27b-it'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.61. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-27b",
    "bench": "zebralogic-zeroeval",
    "score": 0.163,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-27b-it'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "cruxeval-zeroeval",
    "score": 0.46,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-9b-it'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gsm8k-zeroeval",
    "score": 0.8741,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-9b-it'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7282,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-9b-it'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.76. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "zebralogic-zeroeval",
    "score": 0.128,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-9b-it'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "cruxeval-zeroeval",
    "score": 0.5475,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-3.5-turbo-0125'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "gsm8k-zeroeval",
    "score": 0.8036,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-3.5-turbo-0125'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6836,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-3.5-turbo-0125'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "zebralogic-zeroeval",
    "score": 0.101,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-3.5-turbo-0125'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-0314",
    "bench": "cruxeval-zeroeval",
    "score": 0.745,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-0314'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-0314",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8164,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-0314'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-0314",
    "bench": "zebralogic-zeroeval",
    "score": 0.271,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-0314'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "cruxeval-zeroeval",
    "score": 0.7888,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-turbo-2024-04-09'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8531,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-turbo-2024-04-09'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "zebralogic-zeroeval",
    "score": 0.284,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-turbo-2024-04-09'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "cruxeval-zeroeval",
    "score": 0.8612,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-05-13'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "gsm8k-zeroeval",
    "score": 0.9538,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-05-13'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8801,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-05-13'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "zebralogic-zeroeval",
    "score": 0.282,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-05-13'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=19.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "cruxeval-zeroeval",
    "score": 0.87,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "gsm8k-zeroeval",
    "score": 0.9621,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8826,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "zebralogic-zeroeval",
    "score": 0.317,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=3.60. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "cruxeval-zeroeval",
    "score": 0.7588,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "gsm8k-zeroeval",
    "score": 0.9424,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.815,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.07. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "zebralogic-zeroeval",
    "score": 0.201,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.10. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "grok-2-1212",
    "bench": "cruxeval-zeroeval",
    "score": 0.7525,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'grok-2-1212'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "grok-2-1212",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.874,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'grok-2-1212'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "grok-2-1212",
    "bench": "zebralogic-zeroeval",
    "score": 0.277,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'grok-2-1212'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=3.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "zebralogic-zeroeval",
    "score": 0.81,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-2024-12-17'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-mini",
    "bench": "cruxeval-zeroeval",
    "score": 0.9375,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.38. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-mini",
    "bench": "gsm8k-zeroeval",
    "score": 0.9606,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-mini",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8672,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.32. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-mini",
    "bench": "zebralogic-zeroeval",
    "score": 0.526,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.80. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-preview",
    "bench": "cruxeval-zeroeval",
    "score": 0.9588,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-preview-2024-09-12'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=1.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-preview",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.9284,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-preview-2024-09-12'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.61. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-preview",
    "bench": "zebralogic-zeroeval",
    "score": 0.714,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-preview-2024-09-12'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o3-mini",
    "bench": "zebralogic-zeroeval",
    "score": 0.917,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'o3-mini-2025-01-31-high'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.30. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Highest listed effort of the single o3-mini snapshot; low/medium entries not imported.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "cruxeval-zeroeval",
    "score": 0.3412,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-flash-20240226'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Reka API snapshot dated 2024-02-26, catalogue reka-flash-21b-20240226.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "gsm8k-zeroeval",
    "score": 0.7468,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-flash-20240226'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.45. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Reka API snapshot dated 2024-02-26, catalogue reka-flash-21b-20240226.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6472,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-flash-20240226'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.32. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Reka API snapshot dated 2024-02-26, catalogue reka-flash-21b-20240226.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "zebralogic-zeroeval",
    "score": 0.093,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-flash-20240226'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=18.70. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Identity: Reka API snapshot dated 2024-02-26, catalogue reka-flash-21b-20240226.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "cohere-transcribe-arabic-07-2026",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0133,
    "raw_value": 1.33,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/cohere-transcribe-arabic-07-2026.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 (83a75c1) at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp cohere-transcribe-arabic-07-2026 BF16",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/cohere-transcribe-arabic-07-2026.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 1.33% at BF16 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 1.2-1.47. Upstream CohereLabs/cohere-transcribe-arabic-07-2026@0a8193c converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'Q8_0': 1.34, 'F16': 1.33, 'Q6_K': 1.34, 'Q5_K_M': 1.34, 'Q4_K_M': 1.34}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/cohere-transcribe-arabic-07-2026.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=BF16].err_pct",
      "raw_value": 1.33,
      "upstream_repo": "CohereLabs/cohere-transcribe-arabic-07-2026",
      "upstream_commit": "0a8193c"
    }
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0304,
    "raw_value": 3.04,
    "scale_divisor": 100,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/nemotron-3.5-asr-streaming-0.6b.json",
    "source_id": "speech-stt-transcribecpp-mit",
    "by": "independent",
    "date": "2026-09-15",
    "date_basis": "catalog file first and only commit 2026-09-15 (83a75c1) at pinned HEAD c1fee50 (measured_on not recorded)",
    "engine_label": "transcribe.cpp nemotron-3.5-asr-streaming-0.6b F32",
    "sample_count": 2620,
    "data_license": "MIT",
    "note": "transcribe.cpp catalog/nemotron-3.5-asr-streaming-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2,620 utt.) corpus WER 3.04% at F32 (the full-precision tier; F32 > native BF16 > F16 preference), 95% CI 2.83-3.24. Upstream nvidia/nemotron-3.5-asr-streaming-0.6b@24b151a converted to GGUF and numerically validated against the reference implementation by the transcribe.cpp authors; recipe: greedy, timestamps none, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Batch size 1. Quantized rows (not used): {'F16': 3.04, 'Q8_0': 3.05, 'Q6_K': 3.08, 'Q5_K_M': 3.1, 'Q4_K_M': 3.3}.",
    "evidence": {
      "kind": "json",
      "path": "catalog/nemotron-3.5-asr-streaming-0.6b.json",
      "field": "accuracy_benchmarks[dataset=librispeech,split=test-clean,quant=F32].err_pct",
      "raw_value": 3.04,
      "upstream_repo": "nvidia/nemotron-3.5-asr-streaming-0.6b",
      "upstream_commit": "24b151a"
    }
  },
  {
    "model": "whisper-tiny-en",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.224,
    "raw_value": 22.4,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Whisper.cpp Streaming Tiny",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 12.7,
      "librispeech_test_other_pct": 23.3,
      "tedlium_pct": 16,
      "commonvoice_pct": 37.5
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Whisper.cpp Streaming Tiny', column 'Average' = 22.4% (mean of LibriSpeech test-clean 12.7%, test-other 23.3%, TED-LIUM 16.0%, CommonVoice 37.5%). Identity: engine.py WhisperCppStreamingTinyEngine: pywhispercpp Model('tiny.en') for English (ggml conversion of openai/whisper-tiny.en), 3,000 ms chunks, single_segment=True. Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Whisper.cpp Streaming Tiny",
      "column_label": "Average",
      "raw_value": "22.4%"
    }
  },
  {
    "model": "whisper-base-en",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.198,
    "raw_value": 19.8,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Whisper.cpp Streaming Base",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 11.9,
      "librispeech_test_other_pct": 19.9,
      "tedlium_pct": 14.2,
      "commonvoice_pct": 33.2
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Whisper.cpp Streaming Base', column 'Average' = 19.8% (mean of LibriSpeech test-clean 11.9%, test-other 19.9%, TED-LIUM 14.2%, CommonVoice 33.2%). Identity: engine.py WhisperCppStreamingBaseEngine: pywhispercpp Model('base.en') for English (ggml conversion of openai/whisper-base.en), 3,000 ms chunks, single_segment=True. Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Whisper.cpp Streaming Base",
      "column_label": "Average",
      "raw_value": "19.8%"
    }
  },
  {
    "model": "vosk-small-en-us-0-15",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.184,
    "raw_value": 18.4,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Vosk Small",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 9.9,
      "librispeech_test_other_pct": 21,
      "tedlium_pct": 10.7,
      "commonvoice_pct": 32.1
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Vosk Small', column 'Average' = 18.4% (mean of LibriSpeech test-clean 9.9%, test-other 21.0%, TED-LIUM 10.7%, CommonVoice 32.1%). Identity: engine.py VoskStreamingSmallEngine: vosk Model(model_name='vosk-model-small-en-us-0.15'), KaldiRecognizer streaming. Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Vosk Small",
      "column_label": "Average",
      "raw_value": "18.4%"
    }
  },
  {
    "model": "vosk-en-us-0-22",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.115,
    "raw_value": 11.5,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Vosk Large",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 5.4,
      "librispeech_test_other_pct": 12.7,
      "tedlium_pct": 6.6,
      "commonvoice_pct": 21.4
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Vosk Large', column 'Average' = 11.5% (mean of LibriSpeech test-clean 5.4%, test-other 12.7%, TED-LIUM 6.6%, CommonVoice 21.4%). Identity: engine.py VoskStreamingLargeEngine: vosk Model(model_name='vosk-model-en-us-0.22'), KaldiRecognizer streaming. Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Vosk Large",
      "column_label": "Average",
      "raw_value": "11.5%"
    }
  },
  {
    "model": "moonshine-streaming-tiny",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.239,
    "raw_value": 23.9,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Moonshine Tiny",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 11.8,
      "librispeech_test_other_pct": 28.7,
      "tedlium_pct": 12.5,
      "commonvoice_pct": 42.4
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Moonshine Tiny', column 'Average' = 23.9% (mean of LibriSpeech test-clean 11.8%, test-other 28.7%, TED-LIUM 12.5%, CommonVoice 42.4%). Identity: engine.py MoonshineStreamingTinyEngine: moonshine_voice get_model_for_language('en', ModelArch.TINY_STREAMING), 100 ms chunks; Moonshine AI's own moonshine_voice runtime with its Tiny Streaming model (the catalogue's moonshine-streaming-tiny family member). Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Moonshine Tiny",
      "column_label": "Average",
      "raw_value": "23.9%"
    }
  },
  {
    "model": "moonshine-streaming-small",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.134,
    "raw_value": 13.4,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Moonshine Small",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 7,
      "librispeech_test_other_pct": 15,
      "tedlium_pct": 6.9,
      "commonvoice_pct": 24.8
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Moonshine Small', column 'Average' = 13.4% (mean of LibriSpeech test-clean 7.0%, test-other 15.0%, TED-LIUM 6.9%, CommonVoice 24.8%). Identity: engine.py MoonshineStreamingSmallEngine: moonshine_voice get_model_for_language('en', ModelArch.SMALL_STREAMING), 100 ms chunks; Moonshine AI's own moonshine_voice runtime with its Small Streaming model (the catalogue's moonshine-streaming-small family member). Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Moonshine Small",
      "column_label": "Average",
      "raw_value": "13.4%"
    }
  },
  {
    "model": "moonshine-streaming-medium",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.106,
    "raw_value": 10.6,
    "scale_divisor": 100,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md",
    "source_id": "speech-stt-picovoice-apache",
    "by": "independent",
    "date": "2026-03-19",
    "date_basis": "README at pinned commit 43e7689 (2026-03-19)",
    "engine_label": "Moonshine Medium",
    "data_license": "Apache-2.0",
    "components": {
      "librispeech_test_clean_pct": 5.9,
      "librispeech_test_other_pct": 11.4,
      "tedlium_pct": 6.5,
      "commonvoice_pct": 18.7
    },
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Moonshine Medium', column 'Average' = 10.6% (mean of LibriSpeech test-clean 5.9%, test-other 11.4%, TED-LIUM 6.5%, CommonVoice 18.7%). Identity: engine.py MoonshineStreamingMediumEngine: moonshine_voice get_model_for_language('en', ModelArch.MEDIUM_STREAMING), 100 ms chunks; Moonshine AI's own moonshine_voice runtime with its Medium Streaming model (the catalogue's moonshine-streaming-medium family member). Picovoice's own normalizer (normalizer.py). Picovoice sells Cheetah; only the open-model rows are used (cloud engines are unversioned).",
    "evidence": {
      "kind": "table",
      "section": "Results > English > Streaming Engines Word Error Rate",
      "row_label": "Moonshine Medium",
      "column_label": "Average",
      "raw_value": "10.6%"
    }
  },
  {
    "model": "cohere-transcribe-arabic-07-2026",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.2587,
    "raw_value": 25.87,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2026-07-07",
    "date_basis": "Space commit 403a00c52b first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'CohereLabs/cohere-transcribe-arabic-07-2026': Average WER 25.87% over six Arabic test sets (sada 37.47, commonvoice18 5.82, masc_clean 19.60, masc_noisy 27.07, mgb2 15.54, casablanca 49.71); Average CER 11.80%. HF id CohereLabs/cohere-transcribe-arabic-07-2026. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "CohereLabs/cohere-transcribe-arabic-07-2026",
    "components": {
      "sada": 37.47,
      "commonvoice18": 5.82,
      "masc_clean": 19.6,
      "masc_noisy": 27.07,
      "mgb2": 15.54,
      "casablanca": 49.71
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.2832,
    "raw_value": 28.32,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_LLM_7B': Average WER 28.32% over six Arabic test sets (sada 41.61, commonvoice18 8.75, masc_clean 19.69, masc_noisy 29.29, mgb2 14.13, casablanca 56.46); Average CER 12.52%. HF id omnilingual-asr/omniASR_LLM_7B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_LLM_7B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_LLM_7B",
    "components": {
      "sada": 41.61,
      "commonvoice18": 8.75,
      "masc_clean": 19.69,
      "masc_noisy": 29.29,
      "mgb2": 14.13,
      "casablanca": 56.46
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-3b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.2996,
    "raw_value": 29.96,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_LLM_3B': Average WER 29.96% over six Arabic test sets (sada 46.18, commonvoice18 9.15, masc_clean 19.90, masc_noisy 30.03, mgb2 14.22, casablanca 60.27); Average CER 13.77%. HF id omnilingual-asr/omniASR_LLM_3B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_LLM_3B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_LLM_3B",
    "components": {
      "sada": 46.18,
      "commonvoice18": 9.15,
      "masc_clean": 19.9,
      "masc_noisy": 30.03,
      "mgb2": 14.22,
      "casablanca": 60.27
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-1b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.2996,
    "raw_value": 29.96,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_LLM_1B': Average WER 29.96% over six Arabic test sets (sada 43.84, commonvoice18 9.55, masc_clean 20.03, masc_noisy 30.26, mgb2 15.34, casablanca 60.68); Average CER 13.40%. HF id omnilingual-asr/omniASR_LLM_1B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_LLM_1B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_LLM_1B",
    "components": {
      "sada": 43.84,
      "commonvoice18": 9.55,
      "masc_clean": 20.03,
      "masc_noisy": 30.26,
      "mgb2": 15.34,
      "casablanca": 60.68
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-300m",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3296,
    "raw_value": 32.96,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_LLM_300M': Average WER 32.96% over six Arabic test sets (sada 51.38, commonvoice18 12.03, masc_clean 20.66, masc_noisy 32.45, mgb2 16.58, casablanca 64.64); Average CER 14.84%. HF id omnilingual-asr/omniASR_LLM_300M. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_LLM_300M); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_LLM_300M",
    "components": {
      "sada": 51.38,
      "commonvoice18": 12.03,
      "masc_clean": 20.66,
      "masc_noisy": 32.45,
      "mgb2": 16.58,
      "casablanca": 64.64
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3586,
    "raw_value": 35.86,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2026-07-09",
    "date_basis": "Space commit fe0b2736e2 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'nvidia/nemotron-3.5-asr-streaming-0.6b': Average WER 35.86% over six Arabic test sets (sada 53.06, commonvoice18 15.05, masc_clean 23.62, masc_noisy 35.24, mgb2 18.84, casablanca 69.37); Average CER 15.28%. HF id nvidia/nemotron-3.5-asr-streaming-0.6b. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "nvidia/nemotron-3.5-asr-streaming-0.6b",
    "components": {
      "sada": 53.06,
      "commonvoice18": 15.05,
      "masc_clean": 23.62,
      "masc_noisy": 35.24,
      "mgb2": 18.84,
      "casablanca": 69.37
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-ctc-3b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3778,
    "raw_value": 37.78,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_CTC_3B': Average WER 37.78% over six Arabic test sets (sada 69.85, commonvoice18 14.19, masc_clean 21.48, masc_noisy 34.60, mgb2 18.96, casablanca 67.58); Average CER 19.79%. HF id omnilingual-asr/omniASR_CTC_3B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_CTC_3B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_CTC_3B",
    "components": {
      "sada": 69.85,
      "commonvoice18": 14.19,
      "masc_clean": 21.48,
      "masc_noisy": 34.6,
      "mgb2": 18.96,
      "casablanca": 67.58
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-ctc-7b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3812,
    "raw_value": 38.12,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_CTC_7B': Average WER 38.12% over six Arabic test sets (sada 72.69, commonvoice18 12.47, masc_clean 21.08, masc_noisy 35.04, mgb2 20.43, casablanca 67.02); Average CER 20.91%. HF id omnilingual-asr/omniASR_CTC_7B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_CTC_7B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_CTC_7B",
    "components": {
      "sada": 72.69,
      "commonvoice18": 12.47,
      "masc_clean": 21.08,
      "masc_noisy": 35.04,
      "mgb2": 20.43,
      "casablanca": 67.02
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3816,
    "raw_value": 38.16,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'facebook/seamless-m4t-v2-large': Average WER 38.16% over six Arabic test sets (sada 62.52, commonvoice18 21.70, masc_clean 25.04, masc_noisy 33.24, mgb2 20.23, casablanca 66.25); Average CER 17.03%. HF id facebook/seamless-m4t-v2-large. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. SeamlessM4T v2 Large used for Arabic ASR (not in the catalogue before; also on the Quranic and Turkish boards).",
    "engine_label": "facebook/seamless-m4t-v2-large",
    "components": {
      "sada": 62.52,
      "commonvoice18": 21.7,
      "masc_clean": 25.04,
      "masc_noisy": 33.24,
      "mgb2": 20.23,
      "casablanca": 66.25
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-ctc-1b",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.3929,
    "raw_value": 39.29,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_CTC_1B': Average WER 39.29% over six Arabic test sets (sada 71.42, commonvoice18 17.55, masc_clean 22.76, masc_noisy 35.73, mgb2 19.96, casablanca 68.32); Average CER 20.47%. HF id omnilingual-asr/omniASR_CTC_1B. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_CTC_1B); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_CTC_1B",
    "components": {
      "sada": 71.42,
      "commonvoice18": 17.55,
      "masc_clean": 22.76,
      "masc_noisy": 35.73,
      "mgb2": 19.96,
      "casablanca": 68.32
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v2",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.402,
    "raw_value": 40.2,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-large-v2': Average WER 40.20% over six Arabic test sets (sada 57.46, commonvoice18 21.77, masc_clean 27.25, masc_noisy 38.55, mgb2 25.17, casablanca 71.01); Average CER 19.55%. HF id openai/whisper-large-v2. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-large-v2",
    "components": {
      "sada": 57.46,
      "commonvoice18": 21.77,
      "masc_clean": 27.25,
      "masc_noisy": 38.55,
      "mgb2": 25.17,
      "casablanca": 71.01
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4257,
    "raw_value": 42.57,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-large': Average WER 42.57% over six Arabic test sets (sada 63.24, commonvoice18 26.04, masc_clean 28.89, masc_noisy 40.79, mgb2 24.28, casablanca 72.18); Average CER 20.49%. HF id openai/whisper-large. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-large",
    "components": {
      "sada": 63.24,
      "commonvoice18": 26.04,
      "masc_clean": 28.89,
      "masc_noisy": 40.79,
      "mgb2": 24.28,
      "casablanca": 72.18
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-medium",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4557,
    "raw_value": 45.57,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-medium': Average WER 45.57% over six Arabic test sets (sada 67.71, commonvoice18 28.07, masc_clean 29.99, masc_noisy 42.91, mgb2 29.32, casablanca 75.44); Average CER 22.27%. HF id openai/whisper-medium. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-medium",
    "components": {
      "sada": 67.71,
      "commonvoice18": 28.07,
      "masc_clean": 29.99,
      "masc_noisy": 42.91,
      "mgb2": 29.32,
      "casablanca": 75.44
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-ctc-300m",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.4665,
    "raw_value": 46.65,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-11-13",
    "date_basis": "Space commit 3acc25881f first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'omnilingual-asr/omniASR_CTC_300M': Average WER 46.65% over six Arabic test sets (sada 78.11, commonvoice18 27.90, masc_clean 28.40, masc_noisy 43.26, mgb2 26.85, casablanca 75.35); Average CER 21.86%. HF id omnilingual-asr/omniASR_CTC_300M. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Omnilingual ASR v1 checkpoint (fairseq2 card omniASR_CTC_300M); row added 2025-11-13, before the '_v2' December 2025 update, so it is the v1 model.",
    "engine_label": "omnilingual-asr/omniASR_CTC_300M",
    "components": {
      "sada": 78.11,
      "commonvoice18": 27.9,
      "masc_clean": 28.4,
      "masc_noisy": 43.26,
      "mgb2": 26.85,
      "casablanca": 75.35
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "mms-1b-all",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.5454,
    "raw_value": 54.54,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'facebook/mms-1b-all': Average WER 54.54% over six Arabic test sets (sada 77.48, commonvoice18 26.52, masc_clean 38.82, masc_noisy 57.33, mgb2 39.16, casablanca 87.95); Average CER 21.45%. HF id facebook/mms-1b-all. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated. Arabic run of the multilingual MMS-1B-all checkpoint (language adapter selected by the board scripts; the catalogue entry is the same checkpoint).",
    "engine_label": "facebook/mms-1b-all",
    "components": {
      "sada": 77.48,
      "commonvoice18": 26.52,
      "masc_clean": 38.82,
      "masc_noisy": 57.33,
      "mgb2": 39.16,
      "casablanca": 87.95
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-small",
    "bench": "oual-arabic-6set-avg-wer",
    "score": 0.5513,
    "raw_value": 55.13,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py",
    "source_id": "speech-stt-arabic-oual-elm-apache",
    "by": "independent",
    "date": "2025-01-12",
    "date_basis": "Space commit 5769259a07 first published this row's current Average WER (unchanged to pinned revision 667db1e1e4); run date not stated",
    "note": "Row 'openai/whisper-small': Average WER 55.13% over six Arabic test sets (sada 78.02, commonvoice18 24.18, masc_clean 35.93, masc_noisy 56.36, mgb2 48.64, casablanca 87.64); Average CER 21.68%. HF id openai/whisper-small. Added to the board by the Elm maintainer (Space commits by yingzhi) with the open evaluation scripts; per-row runner not stated.",
    "engine_label": "openai/whisper-small",
    "components": {
      "sada": 78.02,
      "commonvoice18": 24.18,
      "masc_clean": 35.93,
      "masc_noisy": 56.36,
      "mgb2": 48.64,
      "casablanca": 87.64
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v2",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.121608,
    "raw_value": 12.1608,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v2' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.12160845731619972 (fraction as published) on 531 recordings. Whisper large-v2 (openai/whisper-large-v2); reproduction agrees with the Betthupferl authors' released scores to within 0.08 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper large-v2",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v2",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.395249,
    "raw_value": 39.5249,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper large-v2' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.395248551752251 (fraction as published) on 3257 recordings. Whisper large-v2 (openai/whisper-large-v2); reproduction agrees with the Betthupferl authors' released scores to within 0.08 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper large-v2",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "omniasr-ctc-1b-v2",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.239267,
    "raw_value": 23.9267,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'OmniASR CTC 1B v2' (run_type additional_benchmark, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.23926679793701733 (fraction as published) on 531 recordings. Label 'OmniASR CTC 1B v2' = fairseq2 card omniASR_CTC_1B_v2; additional full-split benchmark run by the board maintainer (0 blank outputs). Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "engine_label": "OmniASR CTC 1B v2",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "omniasr-ctc-1b-v2",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.671796,
    "raw_value": 67.1796,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'OmniASR CTC 1B v2' (run_type additional_benchmark, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.6717957237968396 (fraction as published) on 3257 recordings. Label 'OmniASR CTC 1B v2' = fairseq2 card omniASR_CTC_1B_v2; additional full-split benchmark run by the board maintainer (0 blank outputs). Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "engine_label": "OmniASR CTC 1B v2",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-medium",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.18339,
    "raw_value": 18.339,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper medium' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.1833899404660905 (fraction as published) on 531 recordings. Whisper medium (openai/whisper-medium); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper medium",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-medium",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.431195,
    "raw_value": 43.1195,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper medium' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.4311953277169178 (fraction as published) on 3257 recordings. Whisper medium (openai/whisper-medium); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper medium",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-small",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.202665,
    "raw_value": 20.2665,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper small' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.20266482582812634 (fraction as published) on 531 recordings. Whisper small (openai/whisper-small); reproduction agrees with the Betthupferl authors' released scores to within 0.01 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper small",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-small",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.473714,
    "raw_value": 47.3714,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper small' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.4737135231313967 (fraction as published) on 3257 recordings. Whisper small (openai/whisper-small); reproduction agrees with the Betthupferl authors' released scores to within 0.01 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper small",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "mms-1b-all",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.298183,
    "raw_value": 29.8183,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'MMS 1B All' (run_type paper_reproduction, status validated, metric_basis mean, 9 blank outputs): standard_audio_wer 0.298183454935937 (fraction as published) on 531 recordings. MMS 1B All (facebook/mms-1b-all); reproduction agrees with the Betthupferl authors' released scores to within 0.05 pp (data/reproducibility-audit.csv). German run of the multilingual MMS-1B-all checkpoint (the catalogue entry is the same checkpoint; language adapter is a run setting).",
    "engine_label": "MMS 1B All",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "mms-1b-all",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.679772,
    "raw_value": 67.9772,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'MMS 1B All' (run_type paper_reproduction, status validated, metric_basis mean, 9 blank outputs): std_reference_wer 0.679772101391856 (fraction as published) on 3257 recordings. MMS 1B All (facebook/mms-1b-all); reproduction agrees with the Betthupferl authors' released scores to within 0.05 pp (data/reproducibility-audit.csv). German run of the multilingual MMS-1B-all checkpoint (the catalogue entry is the same checkpoint; language adapter is a run setting).",
    "engine_label": "MMS 1B All",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-base",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.337897,
    "raw_value": 33.7897,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper base' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.3378967330258139 (fraction as published) on 531 recordings. Whisper base (openai/whisper-base); reproduction agrees with the Betthupferl authors' released scores to within 0.01 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper base",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-base",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.64074,
    "raw_value": 64.074,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper base' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.6407403143875275 (fraction as published) on 3257 recordings. Whisper base (openai/whisper-base); reproduction agrees with the Betthupferl authors' released scores to within 0.01 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper base",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "whisper-tiny",
    "bench": "betthupferl-de-standard-audio-wer",
    "score": 0.444004,
    "raw_value": 44.4004,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper tiny' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): standard_audio_wer 0.4440041377224224 (fraction as published) on 531 recordings. Whisper tiny (openai/whisper-tiny); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper tiny",
    "sample_count": 531,
    "data_license": "MIT"
  },
  {
    "model": "whisper-tiny",
    "bench": "betthupferl-de-dialect-to-standard-wer",
    "score": 0.738428,
    "raw_value": 73.8428,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv",
    "source_id": "speech-stt-bavarian-betthupferl-mit",
    "by": "independent",
    "date": "2026-08-24",
    "date_basis": "Space commit 1f320791b2 (2026-08-24, 'Document benchmark protocol and correct reproduction metrics') published these values; unchanged to pinned revision 1a8d0ac6 (2026-09-21)",
    "note": "Row 'Whisper tiny' (run_type paper_reproduction, status validated, metric_basis mean, 0 blank outputs): std_reference_wer 0.7384284264337335 (fraction as published) on 3257 recordings. Whisper tiny (openai/whisper-tiny); reproduction agrees with the Betthupferl authors' released scores to within 0.02 pp (data/reproducibility-audit.csv).",
    "engine_label": "Whisper tiny",
    "sample_count": 3257,
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.073,
    "raw_value": 7.3,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-26",
    "date_basis": "run job id ElevenLabs_2025-10-26_23-47-14 in results.csv",
    "note": "model_id 'elevenlabs/scribe_v1' (ElevenLabs API model id): Average WER 7.30% = mean of google_chilean_spanish 3.3% (4,374 utt), common_voice 2.21% (152), datarisas 16.4% (50), recomputed as the Space's load_results() does (round 2). Job ElevenLabs_2025-10-26_23-47-14. API model id scribe_v1 (ElevenLabs job), run 2025-10-26 when scribe_v1 was still served.",
    "engine_label": "elevenlabs/scribe_v1",
    "sample_count": 4576,
    "components": {
      "datarisas": 16.4,
      "google_chilean_spanish": 3.3,
      "common_voice": 2.21
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "phi-4-multimodal-instruct",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.0945,
    "raw_value": 9.45,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id Phi4Multimodal_2025-10-27_00-32-26 in results.csv",
    "note": "model_id 'microsoft/Phi-4-multimodal-instruct' (exact HF id): Average WER 9.45% = mean of google_chilean_spanish 4.44% (4,374 utt), common_voice 3.25% (152), datarisas 20.67% (50), recomputed as the Space's load_results() does (round 2). Job Phi4Multimodal_2025-10-27_00-32-26.",
    "engine_label": "microsoft/Phi-4-multimodal-instruct",
    "sample_count": 4576,
    "components": {
      "datarisas": 20.67,
      "google_chilean_spanish": 4.44,
      "common_voice": 3.25
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-small",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.1638,
    "raw_value": 16.38,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-10-27",
    "date_basis": "run job id Transformers_2025-10-27_00-59-35 in results.csv",
    "note": "model_id 'openai/whisper-small' (exact HF id): Average WER 16.38% = mean of google_chilean_spanish 7.99% (4,374 utt), common_voice 10.34% (152), datarisas 30.8% (50), recomputed as the Space's load_results() does (round 2). Job Transformers_2025-10-27_00-59-35.",
    "engine_label": "openai/whisper-small",
    "sample_count": 4576,
    "components": {
      "datarisas": 30.8,
      "google_chilean_spanish": 7.99,
      "common_voice": 10.34
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "cl-spanish-asr-3set-avg-wer",
    "score": 0.1477,
    "raw_value": 14.77,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv",
    "source_id": "speech-stt-cl-spanish-asr-apache",
    "by": "independent",
    "date": "2025-11-10",
    "date_basis": "run job id Omnilingual_2025-11-10_23-37-32 in results.csv",
    "note": "model_id 'facebookresearch/omniASR_LLM_7B' (exact HF id): Average WER 14.77% = mean of google_chilean_spanish 5.09% (4,374 utt), common_voice 4.16% (152), datarisas 35.07% (50), recomputed as the Space's load_results() does (round 2). Job Omnilingual_2025-11-10_23-37-32. Omnilingual ASR v1 checkpoint omniASR_LLM_7B (run 2025-11-10, before the December 2025 '_v2' update).",
    "engine_label": "facebookresearch/omniASR_LLM_7B",
    "sample_count": 4576,
    "components": {
      "datarisas": 35.07,
      "google_chilean_spanish": 5.09,
      "common_voice": 4.16
    },
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-large-v2",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.085138,
    "raw_value": 8.5138,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T121622Z (run.json status success)",
    "note": "model_id 'whisper-large-v2' = HF openai/whisper-large-v2 @ ae4642769ce2ad8fc292556ccea8e901f1530655 (transformers-whisper, float16, language fa). Corpus CER 8.51% (95% CI 7.18-10.35); WER 25.76% (95% CI 24.17-27.79); 871/871 recordings; run fleurs-fa-ir-v1-whisper-large-v2-20260817T121622Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.07182985994923383,
      0.10346212527655957
    ],
    "sample_count": 871,
    "engine_label": "whisper-large-v2",
    "data_license": "Apache-2.0",
    "published_value": 0.08513829942924045,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-medium",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.169703,
    "raw_value": 16.9703,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T134655Z (run.json status success)",
    "note": "model_id 'whisper-medium' = HF openai/whisper-medium @ abdf7c39ab9d0397620ccaea8974cc764cd0953e (transformers-whisper, float16, language fa). Corpus CER 16.97% (95% CI 13.37-21.17); WER 43.16% (95% CI 37.72-49.50); 871/871 recordings; run fleurs-fa-ir-v1-whisper-medium-20260817T134655Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.13372676919015164,
      0.21170811410162868
    ],
    "sample_count": 871,
    "engine_label": "whisper-medium",
    "data_license": "Apache-2.0",
    "published_value": 0.16970340016586175,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.209144,
    "raw_value": 20.9144,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-12",
    "date_basis": "run_id timestamp 20260812T004826Z (run.json status success)",
    "note": "model_id 'qwen3-asr-0-6b' = HF Qwen/Qwen3-ASR-0.6B-hf @ 7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c (transformers-qwen, bfloat16, language fa). Corpus CER 20.91% (95% CI 20.22-21.64); WER 48.11% (95% CI 47.04-49.15); 871/871 recordings; run fleurs-fa-ir-v1-qwen3-asr-0-6b-20260812T004826Z, image ghcr.io/armanjr/peste-benchmark:2.0.0. Qwen/Qwen3-ASR-0.6B-hf is the transformers-format release of Qwen3-ASR-0.6B.",
    "ci95": [
      0.20215079267404562,
      0.2163963183746065
    ],
    "sample_count": 871,
    "engine_label": "qwen3-asr-0-6b",
    "data_license": "Apache-2.0",
    "published_value": 0.20914434850480512,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-small",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.25961,
    "raw_value": 25.961,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T152515Z (run.json status success)",
    "note": "model_id 'whisper-small' = HF openai/whisper-small @ 973afd24965f72e36ca33b3055d56a652f456b4d (transformers-whisper, float16, language fa). Corpus CER 25.96% (95% CI 22.07-30.30); WER 61.30% (95% CI 57.39-65.78); 871/871 recordings; run fleurs-fa-ir-v1-whisper-small-20260817T152515Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.22071516177383435,
      0.3029611021262942
    ],
    "sample_count": 871,
    "engine_label": "whisper-small",
    "data_license": "Apache-2.0",
    "published_value": 0.25961022488901897,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-base",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.459035,
    "raw_value": 45.9035,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T105009Z (run.json status success)",
    "note": "model_id 'whisper-base' = HF openai/whisper-base @ e37978b90ca9030d5170a5c07aadb050351a65bb (transformers-whisper, float16, language fa). Corpus CER 45.90% (95% CI 40.66-51.40); WER 102.95% (95% CI 95.05-111.26); 871/871 recordings; run fleurs-fa-ir-v1-whisper-base-20260817T105009Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.40660996521855686,
      0.514044243298918
    ],
    "sample_count": 871,
    "engine_label": "whisper-base",
    "data_license": "Apache-2.0",
    "published_value": 0.45903458705302697,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "whisper-tiny",
    "bench": "peste-fleurs-fa-cer",
    "score": 0.972243,
    "raw_value": 97.2243,
    "source": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv",
    "source_id": "speech-stt-peste-apache",
    "by": "independent",
    "date": "2026-08-17",
    "date_basis": "run_id timestamp 20260817T174950Z (run.json status success)",
    "note": "model_id 'whisper-tiny' = HF openai/whisper-tiny @ 169d4a4341b33bc18d8881c4b69c2e104e1cc0af (transformers-whisper, float16, language fa). Corpus CER 97.22% (95% CI 87.27-108.04); WER 165.46% (95% CI 151.32-180.62); 871/871 recordings; run fleurs-fa-ir-v1-whisper-tiny-20260817T174950Z, image ghcr.io/armanjr/peste-benchmark:2.1.0.",
    "ci95": [
      0.8726679197487697,
      1.0803739160197023
    ],
    "sample_count": 871,
    "engine_label": "whisper-tiny",
    "data_license": "Apache-2.0",
    "published_value": 0.972242548416996,
    "published_unit": "fraction",
    "scale_divisor": 100
  },
  {
    "model": "omniasr-llm-3b",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.1134,
    "raw_value": 11.34,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/omniASR-LLM-3B.json",
    "note": "Row 'omniASR-LLM-3B' (offline). hf_repo facebook/omniASR-LLM-3B, config lang=arb_Arab, language=arb_Arab, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 11.34% (CER 7.86%, alef-insensitive WER 10.14%). Subsets: EveryAyah 4.91%, QUL Al-Nufais 8.44%, phone tlog 26.36%. No biasing or per-model tuning. Omnilingual ASR v1 checkpoint (HF repo facebook/omniASR-*, not the _v2 update).",
    "components": {
      "everyayah_heldout": 4.91,
      "qul_alnufais": 8.44,
      "tlog_holdout": 26.36
    },
    "sample_count": 600,
    "engine_label": "omniASR-LLM-3B",
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-300m",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.2213,
    "raw_value": 22.13,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/omniASR-LLM-300M.json",
    "note": "Row 'omniASR-LLM-300M' (offline). hf_repo facebook/omniASR-LLM-300M, config lang=arb_Arab, language=arb_Arab, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 22.13% (CER 14.93%, alef-insensitive WER 20.85%). Subsets: EveryAyah 9.42%, QUL Al-Nufais 33.37%, phone tlog 24.72%. No biasing or per-model tuning. Omnilingual ASR v1 checkpoint (HF repo facebook/omniASR-*, not the _v2 update).",
    "components": {
      "everyayah_heldout": 9.42,
      "qul_alnufais": 33.37,
      "tlog_holdout": 24.72
    },
    "sample_count": 600,
    "engine_label": "omniASR-LLM-300M",
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-1b",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.2486,
    "raw_value": 24.86,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/omniASR-LLM-1B.json",
    "note": "Row 'omniASR-LLM-1B' (offline). hf_repo facebook/omniASR-LLM-1B, config lang=arb_Arab, language=arb_Arab, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 24.86% (CER 18.77%, alef-insensitive WER 23.5%). Subsets: EveryAyah 9.5%, QUL Al-Nufais 32.8%, phone tlog 37.02%. No biasing or per-model tuning. Omnilingual ASR v1 checkpoint (HF repo facebook/omniASR-*, not the _v2 update).",
    "components": {
      "everyayah_heldout": 9.5,
      "qul_alnufais": 32.8,
      "tlog_holdout": 37.02
    },
    "sample_count": 600,
    "engine_label": "omniASR-LLM-1B",
    "data_license": "Apache-2.0"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.2585,
    "raw_value": 25.85,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/seamless-m4t-v2-large.json",
    "note": "Row 'seamless-m4t-v2-large' (offline). hf_repo facebook/seamless-m4t-v2-large, config tgt_lang=arb, task s2tt/asr, language=arb, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 25.85% (CER 15.87%, alef-insensitive WER 23.98%). Subsets: EveryAyah 18.97%, QUL Al-Nufais 30.49%, phone tlog 29.59%. No biasing or per-model tuning.",
    "components": {
      "everyayah_heldout": 18.97,
      "qul_alnufais": 30.49,
      "tlog_holdout": 29.59
    },
    "sample_count": 600,
    "engine_label": "seamless-m4t-v2-large",
    "data_license": "Apache-2.0"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.3398,
    "raw_value": 33.98,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/omniASR-LLM-7B.json",
    "note": "Row 'omniASR-LLM-7B' (offline). hf_repo facebook/omniASR-LLM-7B, config lang=arb_Arab, language=arb_Arab, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 33.98% (CER 25.54%, alef-insensitive WER 32.79%). Subsets: EveryAyah 26.24%, QUL Al-Nufais 46.01%, phone tlog 27.28%. No biasing or per-model tuning. Omnilingual ASR v1 checkpoint (HF repo facebook/omniASR-*, not the _v2 update).",
    "components": {
      "everyayah_heldout": 26.24,
      "qul_alnufais": 46.01,
      "tlog_holdout": 27.28
    },
    "sample_count": 600,
    "engine_label": "omniASR-LLM-7B",
    "data_license": "Apache-2.0"
  },
  {
    "model": "mms-1b-all",
    "bench": "quranic-asr-heldout600-wer",
    "score": 0.4549,
    "raw_value": 45.49,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/results.json",
    "source_id": "speech-stt-quranic-asr-lb-apache",
    "by": "independent",
    "date": "2026-08-15",
    "date_basis": "rescore against benchmark v1.1 references (Space commit 7501f0b, 2026-08-15); transcription run_date 2026-07-02 per predictions/mms-1b-all.json",
    "note": "Row 'mms-1b-all' (offline). hf_repo facebook/mms-1b-all, config adapter=ara (\"target lang adapter loaded\"), language=ara, re-run from scratch. Overall WER on all 600 held-out clips (benchmark v1.1, official score.py normalisation: diacritics and Quranic signs stripped, alef/ya/ta-marbuta folded): 45.49% (CER 11.45%, alef-insensitive WER 41.57%). Subsets: EveryAyah 40.94%, QUL Al-Nufais 50.86%, phone tlog 44.25%. No biasing or per-model tuning. Arabic adapter of the multilingual MMS-1B-all checkpoint (same checkpoint as the catalogue entry).",
    "components": {
      "everyayah_heldout": 40.94,
      "qul_alnufais": 50.86,
      "tlog_holdout": 44.25
    },
    "sample_count": 600,
    "engine_label": "mms-1b-all",
    "data_license": "Apache-2.0"
  },
  {
    "model": "whisper-tiny",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.4977,
    "raw_value": 49.77,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'tiny', hf_id openai/whisper-tiny. Overall corpus WER 0.4977 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.5065; CER 0.2089; KDHO 0.4598. Run by the leaderboard maintainer.",
    "engine_label": "tiny",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.6796,
      "fleurs_tr": 0.459,
      "khanacademy_turkish": 0.4013,
      "mediaspeech_tr": 0.4943,
      "real_turnturk": 0.9611,
      "voxforge_tr": 0.4581,
      "youtube_video_transkriptleri_tr": 0.4608
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-base",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.3455,
    "raw_value": 34.55,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'base', hf_id openai/whisper-base. Overall corpus WER 0.3455 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.3622; CER 0.1404; KDHO 0.3122. Run by the leaderboard maintainer.",
    "engine_label": "base",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.4799,
      "fleurs_tr": 0.2891,
      "khanacademy_turkish": 0.2565,
      "mediaspeech_tr": 0.3457,
      "real_turnturk": 0.8376,
      "voxforge_tr": 0.2787,
      "youtube_video_transkriptleri_tr": 0.3165
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-small",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.2171,
    "raw_value": 21.71,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'small', hf_id openai/whisper-small. Overall corpus WER 0.2171 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.2367; CER 0.101; KDHO 0.1929. Run by the leaderboard maintainer.",
    "engine_label": "small",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.2632,
      "fleurs_tr": 0.1484,
      "khanacademy_turkish": 0.1527,
      "mediaspeech_tr": 0.2303,
      "real_turnturk": 0.6606,
      "voxforge_tr": 0.1462,
      "youtube_video_transkriptleri_tr": 0.1945
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-medium",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.1605,
    "raw_value": 16.05,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'medium', hf_id openai/whisper-medium. Overall corpus WER 0.1605 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.18; CER 0.0821; KDHO 0.1422. Run by the leaderboard maintainer.",
    "engine_label": "medium",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.1595,
      "fleurs_tr": 0.0837,
      "khanacademy_turkish": 0.1081,
      "mediaspeech_tr": 0.1577,
      "real_turnturk": 0.6262,
      "voxforge_tr": 0.0902,
      "youtube_video_transkriptleri_tr": 0.1481
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v2",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.1532,
    "raw_value": 15.32,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'large-v2', hf_id openai/whisper-large-v2. Overall corpus WER 0.1532 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.1721; CER 0.089; KDHO 0.1373. Run by the leaderboard maintainer.",
    "engine_label": "large-v2",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.1218,
      "fleurs_tr": 0.0625,
      "khanacademy_turkish": 0.0922,
      "mediaspeech_tr": 0.1657,
      "real_turnturk": 0.6399,
      "voxforge_tr": 0.0672,
      "youtube_video_transkriptleri_tr": 0.1437
    },
    "data_license": "MIT"
  },
  {
    "model": "mms-1b-all",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.3151,
    "raw_value": 31.51,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'mms-1b-all', hf_id facebook/mms-1b-all. Overall corpus WER 0.3151 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.3224; CER 0.1049; KDHO 0.2907. Run by the leaderboard maintainer. Turkish language adapter (tur) of the multilingual MMS-1B-all checkpoint (same checkpoint as the catalogue entry).",
    "engine_label": "mms-1b-all",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.304,
      "fleurs_tr": 0.1931,
      "khanacademy_turkish": 0.2389,
      "mediaspeech_tr": 0.3384,
      "real_turnturk": 0.7587,
      "voxforge_tr": 0.1893,
      "youtube_video_transkriptleri_tr": 0.3313
    },
    "data_license": "MIT"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "turkasr-bench-tr7-wer",
    "score": 0.2159,
    "raw_value": 21.59,
    "scale_divisor": 100,
    "source": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json",
    "source_id": "speech-stt-turkasr-bench-mit",
    "by": "independent",
    "date": "2026-09-25",
    "date_basis": "turkasr_bench.json 'version': '2026-09-25' at the pinned Space revision (README: last updated September 2026)",
    "note": "Model key 'seamless-m4t-v2-large', hf_id facebook/seamless-m4t-v2-large. Overall corpus WER 0.2159 (fraction as published) on n=5200 utterances, main 'num' normalization (digits verbalized); raw normalization WER 0.2321; CER 0.1328; KDHO 0.1908. Run by the leaderboard maintainer.",
    "engine_label": "seamless-m4t-v2-large",
    "sample_count": 5200,
    "components": {
      "common_voice_tr": 0.1436,
      "fleurs_tr": 0.0762,
      "khanacademy_turkish": 0.1897,
      "mediaspeech_tr": 0.1074,
      "real_turnturk": 0.5919,
      "voxforge_tr": 0.0908,
      "youtube_video_transkriptleri_tr": 0.3224
    },
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.1466,
    "raw_value": 0.1466,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'elevenlabs_scribe': corpus CER 0.1466 (95% CI 0.1332-0.1609, results/leaderboard_ci.csv), WER 0.2603 companion only, files_scored 1001/1001. ElevenLabs model_id=scribe_v1, language_code=ben (config.yaml). Values as published in the committed leaderboard (not recomputed by us for this row).",
    "ci95": [
      0.1332,
      0.1609
    ],
    "sample_count": 1001,
    "companion_metric": {
      "word_error_rate": 0.2603,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "google-chirp-2",
    "bench": "bangla-riverborn-1001-batch-cer",
    "score": 0.2193,
    "raw_value": 0.2193,
    "raw_unit": "character error ratio (fraction)",
    "source": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv",
    "source_id": "speech-stt-bangla-riverborn-mit",
    "by": "independent",
    "date": "2026-09-24",
    "date_basis": "v1.0 commit date; README: tested September 2026",
    "note": "Batch row 'google_chirp2': corpus CER 0.2193 (95% CI 0.1993-0.241, results/leaderboard_ci.csv), WER 0.4141 companion only, files_scored 1000/1001. Google Cloud Speech-to-Text v2 model=chirp_2, location us-central1, language_code bn-BD (config.yaml; its comment says to also try bn-IN and keep the better one, so the locale used is not certain). Values as published in the committed leaderboard (not recomputed by us for this row).",
    "ci95": [
      0.1993,
      0.241
    ],
    "sample_count": 1000,
    "companion_metric": {
      "word_error_rate": 0.4141,
      "ranked": false
    },
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0198,
    "raw_value": 1.98,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'groq-whisper-v3-turbo': corpus WER 1.98% (95% CI 1.8-2.14), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Groq-hosted model=whisper-large-v3-turbo (OpenAI-compatible /audio/transcriptions; src/cloud_adapters.py PROVIDERS); provider-hosted open weights, runtime/precision not disclosed.",
    "ci95_percent": [
      1.8,
      2.14
    ],
    "sample_count": 2620,
    "engine_label": "groq-whisper-v3-turbo",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0199,
    "raw_value": 1.99,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'elevenlabs-scribe': corpus WER 1.99% (95% CI 1.81-2.18), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. ElevenLabs speech-to-text API; adapter default model_id='scribe_v1' (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      1.81,
      2.18
    ],
    "sample_count": 2620,
    "engine_label": "elevenlabs-scribe",
    "data_license": "MIT"
  },
  {
    "model": "google-chirp-2",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0232,
    "raw_value": 2.32,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'google-chirp': corpus WER 2.32% (95% CI 2.14-2.49), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Google Cloud Speech-to-Text v2; adapter default model='chirp_2' (global) (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      2.14,
      2.49
    ],
    "sample_count": 2620,
    "engine_label": "google-chirp",
    "data_license": "MIT"
  },
  {
    "model": "revai-machine",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0444,
    "raw_value": 4.44,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'rev': corpus WER 4.44% (95% CI 4.21-4.69), scored 2582 of 2620 utterances (98.5% coverage), 52871 reference words. Rev.ai async API, transcriber='machine' (src/cloud_adapters.py PROVIDERS); unversioned service tier, as the catalogue entry.",
    "ci95_percent": [
      4.21,
      4.69
    ],
    "sample_count": 2582,
    "engine_label": "rev",
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "verli-earnings22-wer",
    "score": 0.119,
    "raw_value": 11.9,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'groq-whisper-v3-turbo': corpus WER 11.9% (95% CI 11.43-12.4), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Groq-hosted model=whisper-large-v3-turbo (OpenAI-compatible /audio/transcriptions; src/cloud_adapters.py PROVIDERS); provider-hosted open weights, runtime/precision not disclosed.",
    "ci95_percent": [
      11.43,
      12.4
    ],
    "sample_count": 2741,
    "engine_label": "groq-whisper-v3-turbo",
    "data_license": "MIT"
  },
  {
    "model": "google-chirp-2",
    "bench": "verli-earnings22-wer",
    "score": 0.1267,
    "raw_value": 12.67,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'google-chirp': corpus WER 12.67% (95% CI 12.19-13.19), scored 2737 of 2741 utterances (99.9% coverage), 48883 reference words. Google Cloud Speech-to-Text v2; adapter default model='chirp_2' (global) (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      12.19,
      13.19
    ],
    "sample_count": 2737,
    "engine_label": "google-chirp",
    "data_license": "MIT"
  },
  {
    "model": "whisper-small",
    "bench": "verli-earnings22-wer",
    "score": 0.1492,
    "raw_value": 14.92,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'whisper-small': corpus WER 14.92% (95% CI 14.27-15.57), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. On-device via WhisperKit (whisperkit-cli --model whisper-small, i.e. the openai_whisper-small Core ML conversion of openai/whisper-small; --language en --chunking-strategy none; src/run_ood_engines.py); not the PyTorch reference runtime.",
    "ci95_percent": [
      14.27,
      15.57
    ],
    "sample_count": 2741,
    "engine_label": "whisper-small",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "verli-earnings22-wer",
    "score": 0.1544,
    "raw_value": 15.44,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'elevenlabs-scribe': corpus WER 15.44% (95% CI 14.68-16.23), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. ElevenLabs speech-to-text API; adapter default model_id='scribe_v1' (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      14.68,
      16.23
    ],
    "sample_count": 2741,
    "engine_label": "elevenlabs-scribe",
    "data_license": "MIT"
  },
  {
    "model": "google-chirp-2",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0226,
    "raw_value": 2.26,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'google-chirp': corpus WER 2.26% (95% CI 2.07-2.46), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. Google Cloud Speech-to-Text v2; adapter default model='chirp_2' (global) (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      2.07,
      2.46
    ],
    "sample_count": 1200,
    "engine_label": "google-chirp",
    "data_license": "MIT"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0312,
    "raw_value": 3.12,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'groq-whisper-v3-turbo': corpus WER 3.12% (95% CI 2.83-3.43), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. Groq-hosted model=whisper-large-v3-turbo (OpenAI-compatible /audio/transcriptions; src/cloud_adapters.py PROVIDERS); provider-hosted open weights, runtime/precision not disclosed.",
    "ci95_percent": [
      2.83,
      3.43
    ],
    "sample_count": 1200,
    "engine_label": "groq-whisper-v3-turbo",
    "data_license": "MIT"
  },
  {
    "model": "elevenlabs-scribe-v1",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0317,
    "raw_value": 3.17,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'elevenlabs-scribe': corpus WER 3.17% (95% CI 2.9-3.43), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. ElevenLabs speech-to-text API; adapter default model_id='scribe_v1' (src/cloud_adapters.py PROVIDERS at the pinned commit). The adapter default is env-overridable and the README asks users to confirm some vendor ids on the first live call (it names ElevenLabs Scribe v1/v2 and Google chirp_2/chirp_3); mapped on the committed default, as for the AssemblyAI rows.",
    "ci95_percent": [
      2.9,
      3.43
    ],
    "sample_count": 1200,
    "engine_label": "elevenlabs-scribe",
    "data_license": "MIT"
  },
  {
    "model": "whisper-small",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.041,
    "raw_value": 4.1,
    "scale_divisor": 100,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "source_id": "speech-stt-verli-mit",
    "by": "independent",
    "date": "2026-07-19",
    "date_basis": "results commit 74a8054 (files unchanged to pinned HEAD); README: cloud runs 2026-07-16..18",
    "note": "engine 'whisper-small': corpus WER 4.1% (95% CI 3.52-4.88), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. On-device via WhisperKit (whisperkit-cli --model whisper-small, i.e. the openai_whisper-small Core ML conversion of openai/whisper-small; --language en --chunking-strategy none; src/run_ood_engines.py); not the PyTorch reference runtime.",
    "ci95_percent": [
      3.52,
      4.88
    ],
    "sample_count": 1200,
    "engine_label": "whisper-small",
    "data_license": "MIT"
  },
  {
    "model": "parakeet-tdt-1-1b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-tdt-1.1B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.234,
    "raw_value": 23.4,
    "note": "Table 6 row 'Parakeet-tdt-1.1B': mean WER 23.40% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Nvidia Parakeet-tdt-1.1B'. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 28.65,
      "ghana_wer_percent": 22.08,
      "kenya_wer_percent": 21.21,
      "rwanda_wer_percent": 20.39,
      "uganda_wer_percent": 21.35,
      "south_africa_wer_percent": 25.96,
      "north_africa_wer_percent": 24.13
    }
  },
  {
    "model": "parakeet-tdt-1-1b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Parakeet-tdt-1.1B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.2714,
    "raw_value": 27.14,
    "note": "Table 3 row 'Parakeet-tdt-1.1B', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 27.14%. Identity: Table 2 'Nvidia Parakeet-tdt-1.1B'. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 28.45,
      "dialog_wer_percent": 15.14,
      "med_conv_nig_wer_percent": 29.98,
      "afrinames_wer_percent": 45.66,
      "call_center_wer_percent": 25.26,
      "robustness_wer_percent": 44.62
    }
  },
  {
    "model": "canary-1b-flash",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Canary-1B-flash",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2381,
    "raw_value": 23.81,
    "note": "Table 6 row 'Canary-1B-flash': mean WER 23.81% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Nvidia Canary-1B-flash'. Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 29,
      "ghana_wer_percent": 25.06,
      "kenya_wer_percent": 18.25,
      "rwanda_wer_percent": 20.39,
      "uganda_wer_percent": 23.65,
      "south_africa_wer_percent": 26.3,
      "north_africa_wer_percent": 24.04
    }
  },
  {
    "model": "canary-1b-flash",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Canary-1B-flash",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.1913,
    "raw_value": 19.13,
    "note": "Table 3 row 'Canary-1B-flash', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 19.13%. Identity: Table 2 'Nvidia Canary-1B-flash'. The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 29.77,
      "dialog_wer_percent": 48.5,
      "med_conv_nig_wer_percent": 93.62,
      "afrinames_wer_percent": 44.1,
      "call_center_wer_percent": 88.71,
      "robustness_wer_percent": 51.32
    }
  },
  {
    "model": "crisperwhisper",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "CrisperWhisper",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.6618,
    "raw_value": 66.18,
    "note": "Table 6 row 'CrisperWhisper': mean WER 66.18% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Nyra Health CrisperWhisper' (1.54B). Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 74.5,
      "ghana_wer_percent": 80.99,
      "kenya_wer_percent": 74.72,
      "rwanda_wer_percent": 40,
      "uganda_wer_percent": 58.29,
      "south_africa_wer_percent": 72.11,
      "north_africa_wer_percent": 62.64
    }
  },
  {
    "model": "crisperwhisper",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "CrisperWhisper",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.7935,
    "raw_value": 79.35,
    "note": "Table 3 row 'CrisperWhisper', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 79.35%. Identity: Table 2 'Nyra Health CrisperWhisper' (1.54B). The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 63.8,
      "dialog_wer_percent": 72.72,
      "med_conv_nig_wer_percent": 83.12,
      "afrinames_wer_percent": 70.14,
      "call_center_wer_percent": 35.52,
      "robustness_wer_percent": 38.82
    }
  },
  {
    "model": "granite-speech-3-3-2b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "IBM Granite-3.3-2B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2554,
    "raw_value": 25.54,
    "note": "Table 6 row 'IBM Granite-3.3-2B': mean WER 25.54% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'IBM Granite-3.3-2B' listed under SpeechLLMs (ibm-granite/granite-speech-3.3-2b). Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 33.05,
      "ghana_wer_percent": 25.27,
      "kenya_wer_percent": 21.33,
      "rwanda_wer_percent": 21.18,
      "uganda_wer_percent": 21.55,
      "south_africa_wer_percent": 26.16,
      "north_africa_wer_percent": 30.25
    }
  },
  {
    "model": "granite-speech-3-3-2b",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "IBM Granite-3.3-2B",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.2067,
    "raw_value": 20.67,
    "note": "Table 3 row 'IBM Granite-3.3-2B', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 20.67%. Identity: Table 2 'IBM Granite-3.3-2B' listed under SpeechLLMs (ibm-granite/granite-speech-3.3-2b). The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 34.38,
      "dialog_wer_percent": 99.59,
      "med_conv_nig_wer_percent": 96.3,
      "afrinames_wer_percent": 49.51,
      "call_center_wer_percent": 27.1,
      "robustness_wer_percent": 45.86
    }
  },
  {
    "model": "phi-4-multimodal-instruct",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Phi-4 MM-Instruct",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-accents7-mean-wer",
    "score": 0.2593,
    "raw_value": 25.93,
    "note": "Table 6 row 'Phi-4 MM-Instruct': mean WER 25.93% over seven African accent regions (NG, GH, KE, RW, UG, ZA, North Africa) of AfriSpeech-MultiBench. Identity: Table 2 'Microsoft Phi-4 MM-Instruct' (5.6B). Zero-shot, default decoding, uniform 16 kHz mono input, model-specific normalisation (§3.3). Run by Intron Health (not the model maker).",
    "components": {
      "nigeria_wer_percent": 27.73,
      "ghana_wer_percent": 18.86,
      "kenya_wer_percent": 27.92,
      "rwanda_wer_percent": 17.25,
      "uganda_wer_percent": 18.49,
      "south_africa_wer_percent": 51.26,
      "north_africa_wer_percent": 20.03
    }
  },
  {
    "model": "phi-4-multimodal-instruct",
    "source": "https://arxiv.org/html/2511.14255v1",
    "source_id": "speech-stt-afrispeech-multibench-report",
    "by": "independent",
    "date": "2025-11-18",
    "date_basis": "arXiv v1 submission date (18 Nov 2025); run date not stated",
    "engine_label": "Phi-4 MM-Instruct",
    "scale_divisor": 100,
    "data_license": "CC BY 4.0",
    "bench": "afrispeech-multibench-parliament-wer",
    "score": 0.3673,
    "raw_value": 36.73,
    "note": "Table 3 row 'Phi-4 MM-Instruct', AfriSpeech-MultiBench column 'Parl' (legal domain, African parliamentary speech; 8,068 clips, 35.9 h per Table 1): WER 36.73%. Identity: Table 2 'Microsoft Phi-4 MM-Instruct' (5.6B). The Open ASR benchmark columns of Table 3 are not used (may be leaderboard numbers). Other AfriSpeech-MultiBench subsets in components.",
    "components": {
      "afrispeech200_wer_percent": 26.48,
      "dialog_wer_percent": 88.91,
      "med_conv_nig_wer_percent": 130.17,
      "afrinames_wer_percent": 44.28,
      "call_center_wer_percent": 24.99,
      "robustness_wer_percent": 122.45
    }
  },
  {
    "model": "phi-4-multimodal-instruct",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1577,
    "raw_value": 15.77,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Phi-4 Multimodal",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Phi-4 Multimodal' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1577 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id microsoft/Phi-4-multimodal-instruct. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.119,
      "semscore_0_100": 72.6,
      "wer_parkinsons": 0.093,
      "wer_als": 0.211,
      "wer_down_syndrome": 0.253,
      "wer_cerebral_palsy": 0.367
    }
  },
  {
    "model": "parakeet-tdt-1-1b",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.1585,
    "raw_value": 15.85,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Parakeet-TDT-1.1B",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Parakeet-TDT-1.1B' (zero-shot, audio only, no prompt): WER printed as the fraction 0.1585 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id nvidia/parakeet-tdt-1.1b. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.114,
      "semscore_0_100": 69.6,
      "wer_parkinsons": 0.088,
      "wer_als": 0.216,
      "wer_down_syndrome": 0.258,
      "wer_cerebral_palsy": 0.385
    }
  },
  {
    "model": "granite-speech-3-3-8b",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.2366,
    "raw_value": 23.66,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Granite-Speech 3.3-8B",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Granite-Speech 3.3-8B' (zero-shot, audio only, no prompt): WER printed as the fraction 0.2366 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id ibm-granite/granite-speech-3.3-8b. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.183,
      "semscore_0_100": 66.9,
      "wer_parkinsons": 0.169,
      "wer_als": 0.304,
      "wer_down_syndrome": 0.302,
      "wer_cerebral_palsy": 0.449
    }
  },
  {
    "model": "wav2vec2-large-robust-ft-libri-960h",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.3264,
    "raw_value": 32.64,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Wav2Vec2-Robust",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Wav2Vec2-Robust' (zero-shot, audio only, no prompt): WER printed as the fraction 0.3264 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id facebook/wav2vec2-large-robust-ft-libri-960h. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.183,
      "semscore_0_100": 60.3,
      "wer_parkinsons": 0.234,
      "wer_als": 0.401,
      "wer_down_syndrome": 0.526,
      "wer_cerebral_palsy": 0.612
    }
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "sap-dysarthric-rated11k-zeroshot-wer",
    "score": 0.393,
    "raw_value": 39.3,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2605.02782v2",
    "source_id": "speech-stt-sap-dysarthric-report",
    "by": "independent",
    "date": "2026-08-18",
    "date_basis": "arXiv v2 submission date (pinned version, 18 Aug 2026); run date not stated (v1 posted 4 May 2026)",
    "engine_label": "Wav2Vec2",
    "sample_count": 11218,
    "data_license": "CC BY 4.0",
    "note": "Table 4 row 'Wav2Vec2' (zero-shot, audio only, no prompt): WER printed as the fraction 0.3930 (raw_value stored in percent) on the SAP rated subset, N=11,218 utterances (Parkinson's, ALS, Down syndrome, cerebral palsy). Table 19 HF model id facebook/wav2vec2-large-960h. SAP protocol: Whisper EnglishTextNormalizer, dual-reference minimum, clipped at 1.0; long samples chunked for 30-s-limited models. Per-etiology WER, CER and SemScore from Table 22 in components.",
    "components": {
      "cer": 0.229,
      "semscore_0_100": 55.4,
      "wer_parkinsons": 0.294,
      "wer_als": 0.486,
      "wer_down_syndrome": 0.598,
      "wer_cerebral_palsy": 0.688
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/assemblyai_universal_3_pro.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "AssemblyAI Universal-3 Pro",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "assemblyai-universal-3-pro",
    "scale_divisor": 100,
    "bench": "voicecodebench-batch-wer",
    "score": 0.043,
    "raw_value": 4.3,
    "note": "Table 5 (v2) batch row 'AssemblyAI Universal-3 Pro': WER 4.3%, CTEM 84.5%, TSR 50.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='universal-3-pro' provider=assemblyai endpoint=https://api.assemblyai.com/v2/transcript mode=batch. Stored: WER.",
    "components": {
      "ctem_percent": 84.5,
      "tsr_percent": 50.3
    }
  },
  {
    "source": "https://arxiv.org/html/2608.28916v2",
    "source_id": "speech-stt-voicecodebench-report",
    "by": "independent",
    "date": "2026-05-15",
    "date_basis": "evaluation_date in the benchmark's committed prediction file baselines/predictions/assemblyai_universal_3_pro.json (HF besimple-ai/voice-code-bench@bef2824)",
    "engine_label": "AssemblyAI Universal-3 Pro",
    "sample_count": 300,
    "data_license": "CC BY 4.0",
    "model": "assemblyai-universal-3-pro",
    "scale_divisor": 100,
    "bench": "voicecodebench-batch-ctem",
    "score": 0.845,
    "raw_value": 84.5,
    "note": "Table 5 (v2) batch row 'AssemblyAI Universal-3 Pro': WER 4.3%, CTEM 84.5%, TSR 50.3% on 300 human-recorded English workplace segments (1,482 audited entities), raw audio only (no hints/biasing). Identity: run_metadata model='universal-3-pro' provider=assemblyai endpoint=https://api.assemblyai.com/v2/transcript mode=batch. Stored: CTEM (canonical token/entity match, higher is better; decided by an audited GPT-5.5 recoverability verifier).",
    "components": {
      "wer_percent": 4.3,
      "tsr_percent": 50.3
    }
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.0876,
    "raw_value": 8.76,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 8.76% for qwen/qwen3-asr-0.6b. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "assemblyai-universal-3-pro",
    "bench": "rw-voice-eq-asr-avg4-wer",
    "score": 0.094,
    "raw_value": 9.4,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg",
    "source_id": "speech-rw-voice-eq-report",
    "by": "independent",
    "date": "2026-07-16",
    "note": "RW-Voice-EQ Bench (Hume AI Research), arXiv:2607.14846v1, CC BY 4.0. Figure 8 (top-10 ASR by average WER), exact printed bar label 9.40% for assemblyai/universal-3-pro. Unweighted mean of corpus WER on four human-verified private English sets (accented 230 clips, emotional 620, background noise/music 460, conversational Hume-DaiKon 14,142 segments), Open ASR Leaderboard normalizer/scorer. Only the top ten of 40 systems are printed."
  },
  {
    "model": "e2-tts-base",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "E2-TTS",
    "sample_count": 193,
    "note": "Table 2 row 'E2-TTS [ 34 ]': BWS intel. 0.198, BWS recon. 0.672, WER 30.8%, PER 66.0%, UTMOS 2.61, Spk. Sim. 0.713, TTSDS|SAP 90.6, TTSDS|LibriTTS 81.9, TTSDS mean 86.3. Identity: E2-TTS (cites the Microsoft E2 TTS paper, whose model was never released; the only public E2 TTS checkpoint is the F5-TTS authors' SWivid E2TTS_Base reproduction).",
    "bench": "sap-voicerecon-bws-reconstruction",
    "score": 1116.74,
    "raw_value": 0.672,
    "raw_unit": "Plackett–Luce log-worth relative to original recording"
  },
  {
    "model": "e2-tts-base",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "E2-TTS",
    "sample_count": 193,
    "note": "Table 2 row 'E2-TTS [ 34 ]': BWS intel. 0.198, BWS recon. 0.672, WER 30.8%, PER 66.0%, UTMOS 2.61, Spk. Sim. 0.713, TTSDS|SAP 90.6, TTSDS|LibriTTS 81.9, TTSDS mean 86.3. Identity: E2-TTS (cites the Microsoft E2 TTS paper, whose model was never released; the only public E2 TTS checkpoint is the F5-TTS authors' SWivid E2TTS_Base reproduction).",
    "bench": "sap-voicerecon-wer",
    "score": 0.308,
    "raw_value": 30.8,
    "raw_unit": "percent",
    "scale_divisor": 100
  },
  {
    "model": "e2-tts-base",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "E2-TTS",
    "sample_count": 193,
    "note": "Table 2 row 'E2-TTS [ 34 ]': BWS intel. 0.198, BWS recon. 0.672, WER 30.8%, PER 66.0%, UTMOS 2.61, Spk. Sim. 0.713, TTSDS|SAP 90.6, TTSDS|LibriTTS 81.9, TTSDS mean 86.3. Identity: E2-TTS (cites the Microsoft E2 TTS paper, whose model was never released; the only public E2 TTS checkpoint is the F5-TTS authors' SWivid E2TTS_Base reproduction).",
    "bench": "sap-voicerecon-ttsds-mean",
    "score": 0.863,
    "raw_value": 86.3,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100
  },
  {
    "model": "maskgct",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MaskGCT",
    "sample_count": 193,
    "note": "Table 2 row 'MaskGCT [ 37 ]': BWS intel. -0.054, BWS recon. 0.141, WER 17.5%, PER 61.7%, UTMOS 2.58, Spk. Sim. 0.627, TTSDS|SAP 90.8, TTSDS|LibriTTS 79.3, TTSDS mean 85.1. Identity: MaskGCT (cites arXiv:2409.00750; the only released MaskGCT checkpoint is amphion/MaskGCT).",
    "bench": "sap-voicerecon-bws-reconstruction",
    "score": 1024.49,
    "raw_value": 0.141,
    "raw_unit": "Plackett–Luce log-worth relative to original recording"
  },
  {
    "model": "maskgct",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MaskGCT",
    "sample_count": 193,
    "note": "Table 2 row 'MaskGCT [ 37 ]': BWS intel. -0.054, BWS recon. 0.141, WER 17.5%, PER 61.7%, UTMOS 2.58, Spk. Sim. 0.627, TTSDS|SAP 90.8, TTSDS|LibriTTS 79.3, TTSDS mean 85.1. Identity: MaskGCT (cites arXiv:2409.00750; the only released MaskGCT checkpoint is amphion/MaskGCT).",
    "bench": "sap-voicerecon-wer",
    "score": 0.175,
    "raw_value": 17.5,
    "raw_unit": "percent",
    "scale_divisor": 100
  },
  {
    "model": "maskgct",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "MaskGCT",
    "sample_count": 193,
    "note": "Table 2 row 'MaskGCT [ 37 ]': BWS intel. -0.054, BWS recon. 0.141, WER 17.5%, PER 61.7%, UTMOS 2.58, Spk. Sim. 0.627, TTSDS|SAP 90.8, TTSDS|LibriTTS 79.3, TTSDS mean 85.1. Identity: MaskGCT (cites arXiv:2409.00750; the only released MaskGCT checkpoint is amphion/MaskGCT).",
    "bench": "sap-voicerecon-ttsds-mean",
    "score": 0.851,
    "raw_value": 85.1,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100
  },
  {
    "model": "styletts-2-libritts",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "StyleTTS2",
    "sample_count": 193,
    "note": "Table 2 row 'StyleTTS2 [ 42 ]': BWS intel. 2.447, BWS recon. -0.788, WER 6.4%, PER 48.3%, UTMOS 3.71, Spk. Sim. 0.371, TTSDS|SAP 80.9, TTSDS|LibriTTS 85.0, TTSDS mean 83.0. Identity: StyleTTS2 used for zero-shot voice cloning (cites the StyleTTS 2 paper); the only multi-speaker zero-shot StyleTTS 2 checkpoint is yl4579/StyleTTS2-LibriTTS (the LJSpeech one is single-speaker).",
    "bench": "sap-voicerecon-bws-reconstruction",
    "score": 863.11,
    "raw_value": -0.788,
    "raw_unit": "Plackett–Luce log-worth relative to original recording"
  },
  {
    "model": "styletts-2-libritts",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "StyleTTS2",
    "sample_count": 193,
    "note": "Table 2 row 'StyleTTS2 [ 42 ]': BWS intel. 2.447, BWS recon. -0.788, WER 6.4%, PER 48.3%, UTMOS 3.71, Spk. Sim. 0.371, TTSDS|SAP 80.9, TTSDS|LibriTTS 85.0, TTSDS mean 83.0. Identity: StyleTTS2 used for zero-shot voice cloning (cites the StyleTTS 2 paper); the only multi-speaker zero-shot StyleTTS 2 checkpoint is yl4579/StyleTTS2-LibriTTS (the LJSpeech one is single-speaker).",
    "bench": "sap-voicerecon-wer",
    "score": 0.064,
    "raw_value": 6.4,
    "raw_unit": "percent",
    "scale_divisor": 100
  },
  {
    "model": "styletts-2-libritts",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "StyleTTS2",
    "sample_count": 193,
    "note": "Table 2 row 'StyleTTS2 [ 42 ]': BWS intel. 2.447, BWS recon. -0.788, WER 6.4%, PER 48.3%, UTMOS 3.71, Spk. Sim. 0.371, TTSDS|SAP 80.9, TTSDS|LibriTTS 85.0, TTSDS mean 83.0. Identity: StyleTTS2 used for zero-shot voice cloning (cites the StyleTTS 2 paper); the only multi-speaker zero-shot StyleTTS 2 checkpoint is yl4579/StyleTTS2-LibriTTS (the LJSpeech one is single-speaker).",
    "bench": "sap-voicerecon-ttsds-mean",
    "score": 0.83,
    "raw_value": 83,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100
  },
  {
    "model": "amphion-vevo",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "Vevo",
    "sample_count": 193,
    "note": "Table 2 row 'Vevo [ 44 ]': BWS intel. 0.030, BWS recon. -0.944, WER 34.8%, PER 66.2%, UTMOS 2.67, Spk. Sim. 0.563, TTSDS|SAP 87.1, TTSDS|LibriTTS 81.2, TTSDS mean 84.2. Identity: Vevo (cites arXiv:2502.07243, the Vevo 1.0 paper; amphion/Vevo 1.0 release; Vevo1.5 and Vevo2 are later separate releases).",
    "bench": "sap-voicerecon-bws-reconstruction",
    "score": 836.01,
    "raw_value": -0.944,
    "raw_unit": "Plackett–Luce log-worth relative to original recording"
  },
  {
    "model": "amphion-vevo",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "Vevo",
    "sample_count": 193,
    "note": "Table 2 row 'Vevo [ 44 ]': BWS intel. 0.030, BWS recon. -0.944, WER 34.8%, PER 66.2%, UTMOS 2.67, Spk. Sim. 0.563, TTSDS|SAP 87.1, TTSDS|LibriTTS 81.2, TTSDS mean 84.2. Identity: Vevo (cites arXiv:2502.07243, the Vevo 1.0 paper; amphion/Vevo 1.0 release; Vevo1.5 and Vevo2 are later separate releases).",
    "bench": "sap-voicerecon-wer",
    "score": 0.348,
    "raw_value": 34.8,
    "raw_unit": "percent",
    "scale_divisor": 100
  },
  {
    "model": "amphion-vevo",
    "source": "https://arxiv.org/html/2606.21343v1",
    "source_id": "speech-sap-voicerecon-ccby",
    "by": "independent",
    "date": "2026-06-19",
    "date_basis": "arXiv v1 submission date (2026-06-19); synthesis/listening dates not stated",
    "engine_label": "Vevo",
    "sample_count": 193,
    "note": "Table 2 row 'Vevo [ 44 ]': BWS intel. 0.030, BWS recon. -0.944, WER 34.8%, PER 66.2%, UTMOS 2.67, Spk. Sim. 0.563, TTSDS|SAP 87.1, TTSDS|LibriTTS 81.2, TTSDS mean 84.2. Identity: Vevo (cites arXiv:2502.07243, the Vevo 1.0 paper; amphion/Vevo 1.0 release; Vevo1.5 and Vevo2 are later separate releases).",
    "bench": "sap-voicerecon-ttsds-mean",
    "score": 0.842,
    "raw_value": 84.2,
    "raw_unit": "TTSDS score (0-100)",
    "scale_divisor": 100
  },
  {
    "model": "cosyvoice2-0-5b",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "CosyVoice 2",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.8575,
    "raw_value": 4.43,
    "note": "Table 1 row 'CosyVoice 2 [ 33 ]': S-NAT 4.430 ± 0.052, S-SPK-SIM 4.241 ± 0.071, S-ACC-SIM 4.097 ± 0.082, O-WER 0.143, O-SPK-SIM 0.724, O-UTMOS 4.349. This column: 4.43 ± 0.052 (95% CI). Identity: CosyVoice 2 (cites arXiv:2412.10117; the only released CosyVoice 2 checkpoint is FunAudioLLM/CosyVoice2-0.5B).",
    "ci95_raw": [
      4.378,
      4.482
    ]
  },
  {
    "model": "cosyvoice2-0-5b",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "CosyVoice 2",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.81025,
    "raw_value": 4.241,
    "note": "Table 1 row 'CosyVoice 2 [ 33 ]': S-NAT 4.430 ± 0.052, S-SPK-SIM 4.241 ± 0.071, S-ACC-SIM 4.097 ± 0.082, O-WER 0.143, O-SPK-SIM 0.724, O-UTMOS 4.349. This column: 4.241 ± 0.071 (95% CI). Identity: CosyVoice 2 (cites arXiv:2412.10117; the only released CosyVoice 2 checkpoint is FunAudioLLM/CosyVoice2-0.5B).",
    "ci95_raw": [
      4.17,
      4.312
    ]
  },
  {
    "model": "maskgct",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "MaskGCT",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.69075,
    "raw_value": 3.763,
    "note": "Table 1 row 'MaskGCT [ 30 ]': S-NAT 3.763 ± 0.070, S-SPK-SIM 4.586 ± 0.056, S-ACC-SIM 4.483 ± 0.061, O-WER 1.556, O-SPK-SIM 0.720, O-UTMOS 3.843. This column: 3.763 ± 0.07 (95% CI). Identity: MaskGCT (cites the MaskGCT paper; the only released checkpoint is amphion/MaskGCT).",
    "ci95_raw": [
      3.693,
      3.833
    ]
  },
  {
    "model": "maskgct",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "MaskGCT",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.8965,
    "raw_value": 4.586,
    "note": "Table 1 row 'MaskGCT [ 30 ]': S-NAT 3.763 ± 0.070, S-SPK-SIM 4.586 ± 0.056, S-ACC-SIM 4.483 ± 0.061, O-WER 1.556, O-SPK-SIM 0.720, O-UTMOS 3.843. This column: 4.586 ± 0.056 (95% CI). Identity: MaskGCT (cites the MaskGCT paper; the only released checkpoint is amphion/MaskGCT).",
    "ci95_raw": [
      4.53,
      4.642
    ]
  },
  {
    "model": "amphion-vevo",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "VevoTTS",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.66525,
    "raw_value": 3.661,
    "note": "Table 1 row 'VevoTTS [ 32 ]': S-NAT 3.661 ± 0.076, S-SPK-SIM 4.484 ± 0.062, S-ACC-SIM 4.348 ± 0.070, O-WER 5.386, O-SPK-SIM 0.686, O-UTMOS 3.874. This column: 3.661 ± 0.076 (95% CI). Identity: VevoTTS, the TTS mode of Vevo (cites the ICLR 2025 Vevo 1.0 paper; amphion/Vevo 1.0 release with its PhoneToVq8192 AR model).",
    "ci95_raw": [
      3.585,
      3.737
    ]
  },
  {
    "model": "amphion-vevo",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "VevoTTS",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.871,
    "raw_value": 4.484,
    "note": "Table 1 row 'VevoTTS [ 32 ]': S-NAT 3.661 ± 0.076, S-SPK-SIM 4.484 ± 0.062, S-ACC-SIM 4.348 ± 0.070, O-WER 5.386, O-SPK-SIM 0.686, O-UTMOS 3.874. This column: 4.484 ± 0.062 (95% CI). Identity: VevoTTS, the TTS mode of Vevo (cites the ICLR 2025 Vevo 1.0 paper; amphion/Vevo 1.0 release with its PhoneToVq8192 AR model).",
    "ci95_raw": [
      4.422,
      4.546
    ]
  },
  {
    "model": "fireredtts-1",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "FireRedTTS",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.65175,
    "raw_value": 3.607,
    "note": "Table 1 row 'FireRedTTS [ 29 ]': S-NAT 3.607 ± 0.076, S-SPK-SIM 3.424 ± 0.096, S-ACC-SIM 3.422 ± 0.099, O-WER 1.173, O-SPK-SIM 0.461, O-UTMOS 4.007. This column: 3.607 ± 0.076 (95% CI). Identity: FireRedTTS (cites the v1 paper arXiv:2409.03283; FireRedTeam/FireRedTTS, not FireRedTTS-1S or -2).",
    "ci95_raw": [
      3.531,
      3.683
    ]
  },
  {
    "model": "fireredtts-1",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "FireRedTTS",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.606,
    "raw_value": 3.424,
    "note": "Table 1 row 'FireRedTTS [ 29 ]': S-NAT 3.607 ± 0.076, S-SPK-SIM 3.424 ± 0.096, S-ACC-SIM 3.422 ± 0.099, O-WER 1.173, O-SPK-SIM 0.461, O-UTMOS 4.007. This column: 3.424 ± 0.096 (95% CI). Identity: FireRedTTS (cites the v1 paper arXiv:2409.03283; FireRedTeam/FireRedTTS, not FireRedTTS-1S or -2).",
    "ci95_raw": [
      3.328,
      3.52
    ]
  },
  {
    "model": "vall-e-x-plachtaa",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "VALL-E-X",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.51825,
    "raw_value": 3.073,
    "note": "Table 1 row 'VALL-E-X [ 9, 26 ]': S-NAT 3.073 ± 0.083, S-SPK-SIM 3.858 ± 0.085, S-ACC-SIM 3.786 ± 0.089, O-WER 7.603, O-SPK-SIM 0.505, O-UTMOS 3.649. This column: 3.073 ± 0.083 (95% CI). Identity: VALL-E-X (reference [26] is the github.com/Plachtaa/VALL-E-X open implementation; Microsoft VALL-E X itself was never released).",
    "ci95_raw": [
      2.99,
      3.156
    ]
  },
  {
    "model": "vall-e-x-plachtaa",
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "engine_label": "VALL-E-X",
    "sample_count": 160,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.7145,
    "raw_value": 3.858,
    "note": "Table 1 row 'VALL-E-X [ 9, 26 ]': S-NAT 3.073 ± 0.083, S-SPK-SIM 3.858 ± 0.085, S-ACC-SIM 3.786 ± 0.089, O-WER 7.603, O-SPK-SIM 0.505, O-UTMOS 3.649. This column: 3.858 ± 0.085 (95% CI). Identity: VALL-E-X (reference [26] is the github.com/Plachtaa/VALL-E-X open implementation; Microsoft VALL-E X itself was never released).",
    "ci95_raw": [
      3.773,
      3.943
    ]
  },
  {
    "model": "parler-tts-large-v1",
    "bench": "instructttseval-en-avg",
    "score": 0.457,
    "raw_value": 45.7,
    "raw_unit": "percent (Avg. of APS/DSD/RP)",
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2506.16381v1",
    "source_id": "speech-instructttseval-ccby",
    "by": "independent",
    "date": "2025-06-19",
    "date_basis": "arXiv v1 submission date (2025-06-19); generation dates not stated",
    "note": "Table 5 (EN subset) row 'Parler-TTS-large': APS 60.0, DSD 45.9, RP 31.2, Avg. 45.7. Identity: Section 4.2 lists Parler-TTS (Lyth and King, 2024) among the open-source systems the authors ran; the only Parler-TTS large release is parler-tts/parler-tts-large-v1 (description-prompted). Gemini-as-a-judge.",
    "engine_label": "Parler-TTS-large",
    "components": {
      "APS": 60,
      "DSD": 45.9,
      "RP": 31.2
    }
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.078156,
    "raw_value": 7.815556,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2607.23938v1",
    "source_id": "speech-qwen-audio3-tts-report",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 4 (CV3-Eval Multilingual Voice Cloning). Derived unweighted mean of the nine original CV3 languages; CER for zh/ja/ko, WER otherwise (Whisper-large-v3 / Paraformer scoring per section 3.3). Per-language cells kept as components, not separate results. Row CosyVoice3-0.5B, listed under 'Open-source Models'; the caption says all non-API comparison systems 'use publicly released open-source models', and the only released CosyVoice 3 weights are FunAudioLLM/Fun-CosyVoice3-0.5B-2512. Marked lab: authors (Alibaba Token Foundry) and CosyVoice maker (Alibaba FunAudioLLM) share a parent company.",
    "components": {
      "zh": 3.89,
      "en": 5.24,
      "ja": 10.4,
      "ko": 12.8,
      "de": 7.41,
      "es": 4.25,
      "fr": 12.9,
      "it": 6.68,
      "ru": 6.77
    }
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.0173,
    "raw_value": 1.73,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column MOSS-TTS-v1.5, run by Boson AI; OpenMOSS-Team/MOSS-TTS-v1.5 (the 8B MossTTSDelay model; 31 supported languages per its card)."
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0611,
    "raw_value": 6.11,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Column MOSS-TTS-v1.5, run by Boson AI; OpenMOSS-Team/MOSS-TTS-v1.5 (the 8B MossTTSDelay model; 31 supported languages per its card)."
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "minimax-ml23-macro-err",
    "score": 0.0378,
    "raw_value": 3.78,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Column MOSS-TTS-v1.5, run by Boson AI; OpenMOSS-Team/MOSS-TTS-v1.5 (the 8B MossTTSDelay model; 31 supported languages per its card)."
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-zh-macro-err",
    "score": 0.0121,
    "raw_value": 1.21,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. Seed-TTS-Eval test-en WER and test-zh CER, macro mean of the 2 languages (WER/CER x100 as printed). Per-language values not published in this table. Column OmniVoice, run by Boson AI; k2-fsa/OmniVoice (600+ supported languages per its README)."
  },
  {
    "model": "omnivoice",
    "bench": "cv3-eval-ml9-macro-err",
    "score": 0.0492,
    "raw_value": 4.92,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. CV3-Eval multilingual voice clone, macro mean of 9 languages (WER/CER x100 as printed). Per-language values not published in this table. Column OmniVoice, run by Boson AI; k2-fsa/OmniVoice (600+ supported languages per its README)."
  },
  {
    "model": "omnivoice",
    "bench": "minimax-ml23-macro-err",
    "score": 0.0298,
    "raw_value": 2.98,
    "scale_divisor": 100,
    "source": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md",
    "source_id": "speech-higgs3-sglang-apache",
    "by": "independent",
    "date": "2026-06-04",
    "note": "Boson AI's Higgs Audio v3 model-card comparison table as contributed by Boson AI staff (Silin Meng, PR #684, commit 0aafd76b) to the Apache-2.0 sglang-omni repository. 'All numbers are reproducible end-to-end with original metrics and normalization'; same reference audio per prompt. MiniMax multilingual test set, macro mean of 23 languages (WER/CER x100 as printed). Per-language values not published in this table. Column OmniVoice, run by Boson AI; k2-fsa/OmniVoice (600+ supported languages per its README)."
  },
  {
    "model": "mgm-omni-tts-2b-0927",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0253,
    "raw_value": 2.53,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/MGM-Omni.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-11-27",
    "note": "UltraEval-Audio replication/MGM-Omni.md, Speech Generation (seed-tts-eval) table, test-en WER (openai/whisper-large-v3): printed '2.53(2.22)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: registry/model/mgm_omni.yaml, the doc's linked model registry, pins every MGM-Omni entry to wcy1122/MGM-Omni-TTS-2B-0927 (the doc's eval command says '--model mgm-omni'; the registry keys are mgm-omni-tts / -zh plus 8-bit and 4-bit variants of the same checkpoint, so precision is taken to be the default full-precision key). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml. The doc's second 'seedtts-vc-en wer' line (68.5) is the SIM value mislabelled and is not used.",
    "source_model_label": "MGM-Omni",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/MGM-Omni.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/model/mgm_omni.yaml",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "mgm-omni-tts-2b-0927",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0127,
    "raw_value": 1.27,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/MGM-Omni.md",
    "source_id": "speech-ultraeval-audio-apache",
    "by": "independent",
    "date": "2025-11-27",
    "note": "UltraEval-Audio replication/MGM-Omni.md, Speech Generation (seed-tts-eval) table, test-zh CER (Paraformer-zh / seaco paraformer large): printed '1.27(1.18)' — first value is OpenBMB's reproduced measurement, the parenthesised official value is NOT used. Identity: registry/model/mgm_omni.yaml, the doc's linked model registry, pins every MGM-Omni entry to wcy1122/MGM-Omni-TTS-2B-0927 (the doc's eval command says '--model mgm-omni'; the registry keys are mgm-omni-tts / -zh plus 8-bit and 4-bit variants of the same checkpoint, so precision is taken to be the default full-precision key). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml. The doc's second 'seedtts-vc-en wer' line (68.5) is the SIM value mislabelled and is not used.",
    "source_model_label": "MGM-Omni",
    "identity_sources": [
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/MGM-Omni.md",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/model/mgm_omni.yaml",
      "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/registry/evaluator/seed_tts_eval.yaml"
    ],
    "scale_divisor": 100
  },
  {
    "model": "dmospeech-2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.01752,
    "raw_value": 1.752,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "lab",
    "date": "2025-07-20",
    "note": "Table 2, row 'DMOSpeech 2 (Student-Only, 4 steps)', column Seed-TTS-en WER = 1.752; the authors' own model: student with the GRPO RL-optimised duration predictor, 4 sampling steps (the configuration Table 1 calls 'DMOSpeech 2 (4 steps)' and uses as the CMOS anchor). Released as yl4579/DMOSpeech2 (model_85000.pt + model_1500.pt). The Teacher-Guided 16-step sampling setting of the same release scores 1.738 and is not used. Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "DMOSpeech 2 (Student-Only, 4 steps)",
    "identity_sources": [
      "https://huggingface.co/yl4579/DMOSpeech2/tree/b2db04bf6fd5147c487149fe173576eec179f910",
      "https://github.com/yl4579/DMOSpeech2"
    ]
  },
  {
    "model": "dmospeech-2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.698,
    "raw_value": 0.698,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "lab",
    "date": "2025-07-20",
    "note": "Table 2, row 'DMOSpeech 2 (Student-Only, 4 steps)', column Seed-TTS-en SIM = 0.698; the authors' own model: student with the GRPO RL-optimised duration predictor, 4 sampling steps (the configuration Table 1 calls 'DMOSpeech 2 (4 steps)' and uses as the CMOS anchor). Released as yl4579/DMOSpeech2 (model_85000.pt + model_1500.pt). The Teacher-Guided 16-step sampling setting of the same release scores 0.699 and is not used. Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "DMOSpeech 2 (Student-Only, 4 steps)",
    "identity_sources": [
      "https://huggingface.co/yl4579/DMOSpeech2/tree/b2db04bf6fd5147c487149fe173576eec179f910",
      "https://github.com/yl4579/DMOSpeech2"
    ]
  },
  {
    "model": "dmospeech-2",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.01527,
    "raw_value": 1.527,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "lab",
    "date": "2025-07-20",
    "note": "Table 2, row 'DMOSpeech 2 (Student-Only, 4 steps)', column Seed-TTS-zh CER = 1.527; the authors' own model: student with the GRPO RL-optimised duration predictor, 4 sampling steps (the configuration Table 1 calls 'DMOSpeech 2 (4 steps)' and uses as the CMOS anchor). Released as yl4579/DMOSpeech2 (model_85000.pt + model_1500.pt). The Teacher-Guided 16-step sampling setting of the same release scores 1.468 and is not used. Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "DMOSpeech 2 (Student-Only, 4 steps)",
    "identity_sources": [
      "https://huggingface.co/yl4579/DMOSpeech2/tree/b2db04bf6fd5147c487149fe173576eec179f910",
      "https://github.com/yl4579/DMOSpeech2"
    ]
  },
  {
    "model": "dmospeech-2",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.76,
    "raw_value": 0.76,
    "source": "https://arxiv.org/html/2507.14988v1",
    "source_id": "speech-dmospeech2-report-bysa",
    "by": "lab",
    "date": "2025-07-20",
    "note": "Table 2, row 'DMOSpeech 2 (Student-Only, 4 steps)', column Seed-TTS-zh SIM = 0.76; the authors' own model: student with the GRPO RL-optimised duration predictor, 4 sampling steps (the configuration Table 1 calls 'DMOSpeech 2 (4 steps)' and uses as the CMOS anchor). Released as yl4579/DMOSpeech2 (model_85000.pt + model_1500.pt). The Teacher-Guided 16-step sampling setting of the same release scores 0.760 and is not used. Whisper-large-v3 (EN) / Paraformer-zh (ZH), WavLM-large speaker-verification SIM-o; resampled to 24 kHz. CC BY-SA 4.0 (share-alike).",
    "source_model_label": "DMOSpeech 2 (Student-Only, 4 steps)",
    "identity_sources": [
      "https://huggingface.co/yl4579/DMOSpeech2/tree/b2db04bf6fd5147c487149fe173576eec179f910",
      "https://github.com/yl4579/DMOSpeech2"
    ]
  },
  {
    "model": "simple-tts",
    "bench": "librispeech-test-clean-cross-hubert-wer",
    "score": 0.0409,
    "raw_value": 4.09,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 2, row 'Simple-TTS', column WER = 4.09; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Table 2 prints '4.09 (3.4*)' for WER and '0.5305 (0.514*)' for SIM-r: the unstarred values are the authors' re-run with the official checkpoint github.com/asappresearch/simple-tts (Appendix A.4, footnote 16; starred values copied, not used). SIM-o 0.5026 has no starred counterpart.",
    "source_model_label": "Simple-TTS",
    "identity_sources": [
      "https://github.com/asappresearch/simple-tts/blob/25eb8be644c3dd5243f88b51dea315da62faf0f6/README.md"
    ]
  },
  {
    "model": "simple-tts",
    "bench": "librispeech-test-clean-cross-wavlm-sim",
    "score": 0.5026,
    "raw_value": 0.5026,
    "source": "https://arxiv.org/html/2406.11427v2",
    "source_id": "speech-ditto-tts-report-ccby",
    "by": "independent",
    "date": "2025-02-17",
    "note": "Table 2, row 'Simple-TTS', column SIM-o = 0.5026; English-only cross-sentence task, LibriSpeech test-clean 4-10 s clips with a 3-s prompt from another utterance (VALL-E protocol); WER from CTC HuBERT-Large (facebook/hubert-large-ls960-ft) with NeMo text normalisation; SIM-o from WavLM-TDCNN. Table 2 prints '4.09 (3.4*)' for WER and '0.5305 (0.514*)' for SIM-r: the unstarred values are the authors' re-run with the official checkpoint github.com/asappresearch/simple-tts (Appendix A.4, footnote 16; starred values copied, not used). SIM-o 0.5026 has no starred counterpart.",
    "source_model_label": "Simple-TTS",
    "identity_sources": [
      "https://github.com/asappresearch/simple-tts/blob/25eb8be644c3dd5243f88b51dea315da62faf0f6/README.md"
    ]
  },
  {
    "model": "whisper-large",
    "bench": "afrispeech200-test108-wer",
    "score": 0.306,
    "raw_value": 0.306,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "openai/whisper-large",
    "sample_count": 6346,
    "note": "Table 4 row 'openai/whisper-large' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.240, Clinical 0.375, Both 0.306 (stored); Dev (45 accents) Both 0.261; LibriSpeech test-clean 0.167 as run by the authors. Identity: HF openai/whisper-large, the original Whisper large (v1) repository (large-v2 is a separate repo). Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.24,
      "test_clinical": 0.375,
      "dev_both": 0.261,
      "librispeech_test_clean": 0.167
    }
  },
  {
    "model": "whisper-medium",
    "bench": "afrispeech200-test108-wer",
    "score": 0.332,
    "raw_value": 0.332,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "openai/whisper-medium",
    "sample_count": 6346,
    "note": "Table 4 row 'openai/whisper-medium' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.276, Clinical 0.392, Both 0.332 (stored); Dev (45 accents) Both 0.273; LibriSpeech test-clean 0.166 as run by the authors. Identity: HF openai/whisper-medium. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.276,
      "test_clinical": 0.392,
      "dev_both": 0.273,
      "librispeech_test_clean": 0.166
    }
  },
  {
    "model": "whisper-medium-en",
    "bench": "afrispeech200-test108-wer",
    "score": 0.358,
    "raw_value": 0.358,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "openai/whisper-medium-en",
    "sample_count": 6346,
    "note": "Table 4 row 'openai/whisper-medium-en' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.304, Clinical 0.414, Both 0.358 (stored); Dev (45 accents) Both 0.291; LibriSpeech test-clean 0.169 as run by the authors. Identity: printed 'openai/whisper-medium-en' = the English-only openai/whisper-medium.en. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.304,
      "test_clinical": 0.414,
      "dev_both": 0.291,
      "librispeech_test_clean": 0.169
    }
  },
  {
    "model": "whisper-small",
    "bench": "afrispeech200-test108-wer",
    "score": 0.391,
    "raw_value": 0.391,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "openai/whisper-small",
    "sample_count": 6346,
    "note": "Table 4 row 'openai/whisper-small' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.330, Clinical 0.455, Both 0.391 (stored); Dev (45 accents) Both 0.343; LibriSpeech test-clean 0.167 as run by the authors. Identity: HF openai/whisper-small. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.33,
      "test_clinical": 0.455,
      "dev_both": 0.343,
      "librispeech_test_clean": 0.167
    }
  },
  {
    "model": "whisper-small-en",
    "bench": "afrispeech200-test108-wer",
    "score": 0.414,
    "raw_value": 0.414,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "openai/whisper-small-en",
    "sample_count": 6346,
    "note": "Table 4 row 'openai/whisper-small-en' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.350, Clinical 0.482, Both 0.414 (stored); Dev (45 accents) Both 0.352; LibriSpeech test-clean 0.167 as run by the authors. Identity: printed 'openai/whisper-small-en' = the English-only openai/whisper-small.en. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.35,
      "test_clinical": 0.482,
      "dev_both": 0.352,
      "librispeech_test_clean": 0.167
    }
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "afrispeech200-test108-wer",
    "score": 0.611,
    "raw_value": 0.611,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "facebook/wav2vec2-large-960h-lv60-self",
    "sample_count": 6346,
    "note": "Table 4 row 'facebook/wav2vec2-large-960h-lv60-self' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.533, Clinical 0.694, Both 0.611 (stored); Dev (45 accents) Both 0.550; LibriSpeech test-clean 0.051 as run by the authors. Identity: HF facebook/wav2vec2-large-960h-lv60-self. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.533,
      "test_clinical": 0.694,
      "dev_both": 0.55,
      "librispeech_test_clean": 0.051
    }
  },
  {
    "model": "hubert-xlarge-ls960-ft",
    "bench": "afrispeech200-test108-wer",
    "score": 0.641,
    "raw_value": 0.641,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "facebook/hubert-xlarge-ls960-ft",
    "sample_count": 6346,
    "note": "Table 4 row 'facebook/hubert-xlarge-ls960-ft' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.562, Clinical 0.725, Both 0.641 (stored); Dev (45 accents) Both 0.571; LibriSpeech test-clean 0.052 as run by the authors. Identity: HF facebook/hubert-xlarge-ls960-ft. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.562,
      "test_clinical": 0.725,
      "dev_both": 0.571,
      "librispeech_test_clean": 0.052
    }
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "afrispeech200-test108-wer",
    "score": 0.717,
    "raw_value": 0.717,
    "raw_unit": "WER fraction as printed",
    "source": "https://arxiv.org/html/2310.00274v1",
    "source_id": "speech-stt-afrispeech200-report-ccby",
    "by": "independent",
    "date": "2023-09-30",
    "date_basis": "arXiv v1 submission date (30 Sep 2023); run date not stated",
    "engine_label": "facebook/wav2vec2-large-960h",
    "sample_count": 6346,
    "note": "Table 4 row 'facebook/wav2vec2-large-960h' (Open-Source SOTA Models, zero-shot): AfriSpeech-200 Test (108 accents, 750 speakers) WER General 0.641, Clinical 0.797, Both 0.717 (stored); Dev (45 accents) Both 0.652; LibriSpeech test-clean 0.062 as run by the authors. Identity: HF facebook/wav2vec2-large-960h. Authors' own zero-shot runs (Intron Health / AfriSpeech team). The low LibriSpeech scores for Whisper (0.17) suggest a transcript normaliser that penalises Whisper's punctuated output; values kept as published.",
    "components": {
      "test_general": 0.641,
      "test_clinical": 0.797,
      "dev_both": 0.652,
      "librispeech_test_clean": 0.062
    }
  },
  {
    "model": "whisper-large-v3",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.101,
    "raw_value": 10.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "independent",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Large-v3 / Original",
    "note": "Table 1 row 'Large-v3' / 'Original': ESB 8-set average WER 10.1% (VP 8.8, AMI 25.9, E22 19.5, GS 11.1, LS-C 2.4, LS-O 5.5, SG 3.3, TED 4.4). Identity: openai/whisper-large-v3 (uncompressed baseline run by the authors). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 8.8,
      "ami": 25.9,
      "e22": 19.5,
      "gs": 11.1,
      "ls_c": 2.4,
      "ls_o": 5.5,
      "sg": 3.3,
      "ted": 4.4
    }
  },
  {
    "model": "lite-whisper-large-v3-acc",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.101,
    "raw_value": 10.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "lab",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Large-v3 / LiteASR (a)",
    "note": "Table 1 row 'Large-v3' / 'LiteASR (a)': ESB 8-set average WER 10.1% (VP 8.7, AMI 25.7, E22 18.9, GS 11.1, LS-C 2.5, LS-O 5.0, SG 3.4, TED 5.1). Identity: LiteASR (a) on Whisper large-v3 = efficient-speech/lite-whisper-large-v3-acc (the HF card lists the same 429M encoder and 10.1 ESB average). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 8.7,
      "ami": 25.7,
      "e22": 18.9,
      "gs": 11.1,
      "ls_c": 2.5,
      "ls_o": 5,
      "sg": 3.4,
      "ted": 5.1
    }
  },
  {
    "model": "lite-whisper-large-v3",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.102,
    "raw_value": 10.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "lab",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Large-v3 / LiteASR (b)",
    "note": "Table 1 row 'Large-v3' / 'LiteASR (b)': ESB 8-set average WER 10.2% (VP 8.4, AMI 28.7, E22 15.8, GS 12.0, LS-C 2.7, LS-O 6.1, SG 3.1, TED 4.8). Identity: LiteASR (b) on Whisper large-v3 = efficient-speech/lite-whisper-large-v3 (card: 377M encoder, 10.2). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 8.4,
      "ami": 28.7,
      "e22": 15.8,
      "gs": 12,
      "ls_c": 2.7,
      "ls_o": 6.1,
      "sg": 3.1,
      "ted": 4.8
    }
  },
  {
    "model": "lite-whisper-large-v3-fast",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.113,
    "raw_value": 11.3,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "lab",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Large-v3 / LiteASR (c)",
    "note": "Table 1 row 'Large-v3' / 'LiteASR (c)': ESB 8-set average WER 11.3% (VP 8.7, AMI 33.4, E22 17.2, GS 12.3, LS-C 2.8, LS-O 7.4, SG 3.5, TED 5.4). Identity: LiteASR (c) on Whisper large-v3 = efficient-speech/lite-whisper-large-v3-fast (card: 308M encoder, 11.3). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 8.7,
      "ami": 33.4,
      "e22": 17.2,
      "gs": 12.3,
      "ls_c": 2.8,
      "ls_o": 7.4,
      "sg": 3.5,
      "ted": 5.4
    }
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.101,
    "raw_value": 10.1,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "independent",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Turbo / Original",
    "note": "Table 1 row 'Turbo' / 'Original': ESB 8-set average WER 10.1% (VP 9.5, AMI 26.8, E22 17.4, GS 11.4, LS-C 2.6, LS-O 5.5, SG 3.8, TED 4.3). Identity: openai/whisper-large-v3-turbo (uncompressed baseline run by the authors). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 9.5,
      "ami": 26.8,
      "e22": 17.4,
      "gs": 11.4,
      "ls_c": 2.6,
      "ls_o": 5.5,
      "sg": 3.8,
      "ted": 4.3
    }
  },
  {
    "model": "lite-whisper-large-v3-turbo-acc",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.102,
    "raw_value": 10.2,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "lab",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Turbo / LiteASR (a)",
    "note": "Table 1 row 'Turbo' / 'LiteASR (a)': ESB 8-set average WER 10.2% (VP 9.0, AMI 27.7, E22 17.0, GS 11.4, LS-C 2.8, LS-O 6.2, SG 3.1, TED 4.5). Identity: LiteASR (a) on Whisper large-v3-turbo = efficient-speech/lite-whisper-large-v3-turbo-acc (card: 421M encoder, 10.2). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 9,
      "ami": 27.7,
      "e22": 17,
      "gs": 11.4,
      "ls_c": 2.8,
      "ls_o": 6.2,
      "sg": 3.1,
      "ted": 4.5
    }
  },
  {
    "model": "whisper-medium",
    "bench": "liteasr-esb8-avg-wer",
    "score": 0.148,
    "raw_value": 14.8,
    "scale_divisor": 100,
    "source": "https://arxiv.org/html/2502.20583v2",
    "source_id": "speech-stt-liteasr-report-ccby",
    "by": "independent",
    "date": "2025-02-27",
    "date_basis": "arXiv v1 submission (27 Feb 2025; v2 is the pinned version); run date not stated",
    "engine_label": "Medium / Original",
    "note": "Table 1 row 'Medium' / 'Original': ESB 8-set average WER 14.8% (VP 8.7, AMI 31.3, E22 25.9, GS 25.9, LS-C 3.9, LS-O 8.8, SG 5.9, TED 8.2). Identity: openai/whisper-medium (uncompressed baseline run by the authors). hf-audio/esb-datasets-test-only-sorted, the authors' own runs (AMI and E22 far above Open ASR Leaderboard values for the same checkpoints, so not copied).",
    "components": {
      "vp": 8.7,
      "ami": 31.3,
      "e22": 25.9,
      "gs": 25.9,
      "ls_c": 3.9,
      "ls_o": 8.8,
      "sg": 5.9,
      "ted": 8.2
    }
  },
  {
    "model": "xtts-v2",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/XTTS_v2.0.2/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "lab",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "XTTS_v2.0.2",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.007 ± 0.25"
    },
    "bench": "zstts-daps-flores-en-cer",
    "score": 0.005425,
    "raw_value": 0.5425,
    "scale_divisor": 100,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/XTTS_v2.0.2/metrics.csv, English row: CER 0.5425% (Whisper large-v3, punctuation removed), UTMOS 4.007 ± 0.25, SECS 0.6423 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'XTTS_v2.0.2': Coqui XTTS-v2 at its v2.0.2 release (coqui/XTTS-v2 branch v2.0.2). The catalogue entry is XTTS-v2 without a pinned point release; this row is the v2.0.2 point release, evaluated by XTTS's first author. Stored: CER."
  },
  {
    "model": "xtts-v2",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/XTTS_v2.0.2/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "lab",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "XTTS_v2.0.2",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.007 ± 0.25"
    },
    "bench": "zstts-daps-flores-en-ecapa2-secs",
    "score": 0.6423,
    "raw_value": 0.6423,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/XTTS_v2.0.2/metrics.csv, English row: CER 0.5425% (Whisper large-v3, punctuation removed), UTMOS 4.007 ± 0.25, SECS 0.6423 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'XTTS_v2.0.2': Coqui XTTS-v2 at its v2.0.2 release (coqui/XTTS-v2 branch v2.0.2). The catalogue entry is XTTS-v2 without a pinned point release; this row is the v2.0.2 point release, evaluated by XTTS's first author. Stored: SECS."
  },
  {
    "model": "yourtts",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/YourTTS_original/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "lab",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "YourTTS_original",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "3.6034 ± 0.29"
    },
    "bench": "zstts-daps-flores-en-cer",
    "score": 0.028736,
    "raw_value": 2.8736,
    "scale_divisor": 100,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/YourTTS_original/metrics.csv, English row: CER 2.8736% (Whisper large-v3, punctuation removed), UTMOS 3.6034 ± 0.29, SECS 0.4621 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'YourTTS_original': the released YourTTS checkpoint, evaluated by its first author (Edresson Casanova). Stored: CER."
  },
  {
    "model": "yourtts",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/YourTTS_original/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "lab",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "YourTTS_original",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "3.6034 ± 0.29"
    },
    "bench": "zstts-daps-flores-en-ecapa2-secs",
    "score": 0.4621,
    "raw_value": 0.4621,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/YourTTS_original/metrics.csv, English row: CER 2.8736% (Whisper large-v3, punctuation removed), UTMOS 3.6034 ± 0.29, SECS 0.4621 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'YourTTS_original': the released YourTTS checkpoint, evaluated by its first author (Edresson Casanova). Stored: SECS."
  },
  {
    "model": "tortoise-tts-v2",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/Tortoise_num_autoregressive_samples_256_diffusion_iterations_200/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "independent",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "Tortoise_num_autoregressive_samples_256_diffusion_iterations_200",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.0883 ± 0.31"
    },
    "bench": "zstts-daps-flores-en-cer",
    "score": 0.010934,
    "raw_value": 1.0934,
    "scale_divisor": 100,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/Tortoise_num_autoregressive_samples_256_diffusion_iterations_200/metrics.csv, English row: CER 1.0934% (Whisper large-v3, punctuation removed), UTMOS 4.0883 ± 0.31, SECS 0.5492 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'Tortoise_num_autoregressive_samples_256_diffusion_iterations_200': TorToiSe with 256 autoregressive samples and 200 diffusion iterations (the high-quality setting). Stored: CER."
  },
  {
    "model": "tortoise-tts-v2",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/Tortoise_num_autoregressive_samples_256_diffusion_iterations_200/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "independent",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "Tortoise_num_autoregressive_samples_256_diffusion_iterations_200",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.0883 ± 0.31"
    },
    "bench": "zstts-daps-flores-en-ecapa2-secs",
    "score": 0.5492,
    "raw_value": 0.5492,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/Tortoise_num_autoregressive_samples_256_diffusion_iterations_200/metrics.csv, English row: CER 1.0934% (Whisper large-v3, punctuation removed), UTMOS 4.0883 ± 0.31, SECS 0.5492 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'Tortoise_num_autoregressive_samples_256_diffusion_iterations_200': TorToiSe with 256 autoregressive samples and 200 diffusion iterations (the high-quality setting). Stored: SECS."
  },
  {
    "model": "styletts-2-libritts",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/StyleTTS2/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "independent",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "StyleTTS2",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.426 ± 0.07"
    },
    "bench": "zstts-daps-flores-en-cer",
    "score": 0.006789,
    "raw_value": 0.6789,
    "scale_divisor": 100,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/StyleTTS2/metrics.csv, English row: CER 0.6789% (Whisper large-v3, punctuation removed), UTMOS 4.426 ± 0.07, SECS 0.4728 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'StyleTTS2': StyleTTS 2 cloning unseen DAPS speakers, which needs the multi-speaker zero-shot checkpoint yl4579/StyleTTS2-LibriTTS (the LJSpeech checkpoint is single-speaker); checkpoint file not named in the repo. Stored: CER."
  },
  {
    "model": "styletts-2-libritts",
    "source": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/Evaluation/StyleTTS2/metrics.csv",
    "source_id": "speech-zstts-eval-casanova-mit",
    "by": "independent",
    "date": "2024-01-28",
    "date_basis": "metrics.csv first committed 2024-01-28 (unchanged values re-committed in 33d73c4 on 2024-02-14); pinned repository HEAD a9086e4 (2024-09-19)",
    "engine_label": "StyleTTS2",
    "sample_count": 240,
    "data_license": "MIT",
    "companion": {
      "utmos": "4.426 ± 0.07"
    },
    "bench": "zstts-daps-flores-en-ecapa2-secs",
    "score": 0.4728,
    "raw_value": 0.4728,
    "note": "ZS-TTS-Evaluation (Edresson Casanova, MIT) Evaluation/StyleTTS2/metrics.csv, English row: CER 0.6789% (Whisper large-v3, punctuation removed), UTMOS 4.426 ± 0.07, SECS 0.4728 (ECAPA2 cosine to the reference). 240 FLORES+ devtest sentences, zero-shot cloning of 20 clean DAPS speakers (3-8 s references). Identity: folder 'StyleTTS2': StyleTTS 2 cloning unseen DAPS speakers, which needs the multi-speaker zero-shot checkpoint yl4579/StyleTTS2-LibriTTS (the LJSpeech checkpoint is single-speaker); checkpoint file not named in the repo. Stored: SECS."
  },
  {
    "model": "espnet-multiblank-transducer-8421",
    "bench": "librispeech-test-clean-wer",
    "score": 0.027999999999999997,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_multi_blank_transducer_asr_model_valid.loss.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-multiblank-transducer-8421",
    "bench": "librispeech-test-other-wer",
    "score": 0.066,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_multi_blank_transducer_asr_model_valid.loss.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-rnn-transducer",
    "bench": "librispeech-test-clean-wer",
    "score": 0.027999999999999997,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_lm_weight0.0_asr_model_valid.loss.ave_10best/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-rnn-transducer",
    "bench": "librispeech-test-other-wer",
    "score": 0.066,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_lm_weight0.0_asr_model_valid.loss.ave_10best/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-hubert-large-conformer-lm",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018000000000000002,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-hubert-large-conformer-lm",
    "bench": "librispeech-test-other-wer",
    "score": 0.037000000000000005,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-wavlm-large-conformer-lm",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018000000000000002,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-wavlm-large-conformer-lm",
    "bench": "librispeech-test-other-wer",
    "score": 0.037000000000000005,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-wav2vec2-large-conformer-lm",
    "bench": "librispeech-test-clean-wer",
    "score": 0.021,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-wav2vec2-large-conformer-lm",
    "bench": "librispeech-test-other-wer",
    "score": 0.051,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_transformer2_en_bpe5000_valid.loss.ave_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-multiconvformer",
    "bench": "librispeech-test-clean-wer",
    "score": 0.022000000000000002,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-multiconvformer",
    "bench": "librispeech-test-other-wer",
    "score": 0.047,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-e-branchformer",
    "bench": "librispeech-test-clean-wer",
    "score": 0.022000000000000002,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-e-branchformer",
    "bench": "librispeech-test-other-wer",
    "score": 0.046,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-branchformer-18layer",
    "bench": "librispeech-test-clean-wer",
    "score": 0.023,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: till70epoch_beam60_ctc0.3/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-branchformer-18layer",
    "bench": "librispeech-test-other-wer",
    "score": 0.053,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: till70epoch_beam60_ctc0.3/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-s4-decoder",
    "bench": "librispeech-test-clean-wer",
    "score": 0.023,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: beam60_ctc0.3/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-s4-decoder",
    "bench": "librispeech-test-other-wer",
    "score": 0.05,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: beam60_ctc0.3/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-2020-lm",
    "bench": "librispeech-test-clean-wer",
    "score": 0.024,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_adam_bpe_valid.loss.ave_asr_model_valid.acc.ave/test_clean. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-conformer-2020-lm",
    "bench": "librispeech-test-other-wer",
    "score": 0.057999999999999996,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_asr_lm_lm_train_lm_adam_bpe_valid.loss.ave_asr_model_valid.acc.ave/test_other. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-transformer-18layer-lm",
    "bench": "librispeech-test-clean-wer",
    "score": 0.025,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2620 utterances, 52576 words. Exact decoding/checkpoint label: decode_test_clean_decode_asr_beam_size20_lm_train_lm_adam_bpe_valid.loss.best_asr_model_valid.acc.best. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "espnet-transformer-18layer-lm",
    "bench": "librispeech-test-other-wer",
    "score": 0.062,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/librispeech/asr1/README.md",
    "by": "lab",
    "note": "ESPnet recipe WER table, 2939 utterances, 52343 words. Exact decoding/checkpoint label: decode_test_other_decode_asr_beam_size20_lm_train_lm_adam_bpe_valid.loss.best_asr_model_valid.acc.best. Err column / 100; no CER/TER or development set duplication. Native recipe text/scoring pipeline, not Open ASR normalisation.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "jasper-10x5-nvidia-dle",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0341,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechRecognition/Jasper/README.md",
    "by": "lab",
    "note": "NVIDIA DGX A100 training accuracy table, mixed precision, greedy decoding without language model. Exact DLE recipe run; not a generic architecture or Open ASR score.",
    "source_id": "speech-nvidia-dle-asr"
  },
  {
    "model": "jasper-10x5-nvidia-dle",
    "bench": "librispeech-test-other-wer",
    "score": 0.0971,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechRecognition/Jasper/README.md",
    "by": "lab",
    "note": "NVIDIA DGX A100 training accuracy table, mixed precision, greedy decoding without language model. Exact DLE recipe run; not a generic architecture or Open ASR score.",
    "source_id": "speech-nvidia-dle-asr"
  },
  {
    "model": "quartznet-15x5-nvidia-dle",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0369,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechRecognition/QuartzNet/README.md",
    "by": "lab",
    "note": "NVIDIA DGX A100 training accuracy table, mixed precision, greedy decoding without language model. Exact DLE recipe run; not a generic architecture or Open ASR score.",
    "source_id": "speech-nvidia-dle-asr"
  },
  {
    "model": "quartznet-15x5-nvidia-dle",
    "bench": "librispeech-test-other-wer",
    "score": 0.1069,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechRecognition/QuartzNet/README.md",
    "by": "lab",
    "note": "NVIDIA DGX A100 training accuracy table, mixed precision, greedy decoding without language model. Exact DLE recipe run; not a generic architecture or Open ASR score.",
    "source_id": "speech-nvidia-dle-asr"
  },
  {
    "model": "vits-ljspeech",
    "bench": "vits2021-ljspeech-mos",
    "score": 0.8574999999999999,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "lab",
    "note": "Table 1: human naturalness MOS 4.43 ± 0.06 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "tacotron2-hifigan-ljspeech",
    "bench": "vits2021-ljspeech-mos",
    "score": 0.6925,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "note": "Table 1: human naturalness MOS 3.77 ± 0.08 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "tacotron2-hifigan-ft-ljspeech",
    "bench": "vits2021-ljspeech-mos",
    "score": 0.8125,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "note": "Table 1: human naturalness MOS 4.25 ± 0.07 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "glow-tts-hifigan-ljspeech",
    "bench": "vits2021-ljspeech-mos",
    "score": 0.7849999999999999,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "note": "Table 1: human naturalness MOS 4.14 ± 0.07 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "glow-tts-hifigan-ft-ljspeech",
    "bench": "vits2021-ljspeech-mos",
    "score": 0.8300000000000001,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "note": "Table 1: human naturalness MOS 4.32 ± 0.07 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "vits-vctk",
    "bench": "vits2021-vctk-mos",
    "score": 0.845,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "lab",
    "note": "Table 3: human naturalness MOS 4.38 ± 0.06 (95% CI), 1–5 scale. Authors generated and rated all systems themselves. Store (MOS − 1) / 4. §3.4: public pretrained acoustic models; fine-tuned HiFi-GAN trained up to 100k steps on Tacotron 2 teacher-forced outputs.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "attngan-coco",
    "bench": "tifa-v1",
    "score": 0.581,
    "source": "https://arxiv.org/html/2303.11897v3",
    "by": "independent",
    "note": "Appendix Table 4, Overall TIFA (mPLUG VQA) column, /100. Benchmark authors generated and evaluated images; 4,081 prompts. TIFA Appendix E AttnGAN row; original COCO text-to-image system.",
    "source_id": "visual-tifa"
  },
  {
    "model": "x-lxmert",
    "bench": "tifa-v1",
    "score": 0.586,
    "source": "https://arxiv.org/html/2303.11897v3",
    "by": "independent",
    "note": "Appendix Table 4, Overall TIFA (mPLUG VQA) column, /100. Benchmark authors generated and evaluated images; 4,081 prompts. TIFA Appendix E X-LXMERT row; visual-token text-to-image generation.",
    "source_id": "visual-tifa"
  },
  {
    "model": "dall-e-mega-min-dalle",
    "bench": "tifa-v1",
    "score": 0.794,
    "source": "https://arxiv.org/html/2303.11897v3",
    "by": "independent",
    "note": "Appendix Table 4, Overall TIFA (mPLUG VQA) column, /100. Benchmark authors generated and evaluated images; 4,081 prompts. TIFA Appendix E explicitly identifies minDALL-E as DALL·E Mini with Mega weights. HPS Appendix D names the same min-dalle implementation, not original OpenAI DALL·E.",
    "source_id": "visual-tifa"
  },
  {
    "model": "dall-e-mega-min-dalle",
    "bench": "hpsv2-benchmark",
    "score": 0.2583,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md",
    "by": "independent",
    "note": "Committed README v2 benchmark Averaged column /100; not v2.1 and no per-style duplication. HPS v2 Appendix D: kuprel/min-dalle reproduction, default hyperparameters except grid_size=1; README identifies Mega weights.",
    "source_id": "visual-hpsv2-readme"
  },
  {
    "model": "fusedream-biggan-512",
    "bench": "hpsv2-benchmark",
    "score": 0.2528,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md",
    "by": "independent",
    "note": "Committed README v2 benchmark Averaged column /100; not v2.1 and no per-style duplication. HPS v2 Appendix D: 512×512 output, 1000 initialisation iterations, 1000 optimisation iterations, 5 basis images.",
    "source_id": "visual-hpsv2-readme"
  },
  {
    "model": "lafite-hps2023",
    "bench": "hpsv2-benchmark",
    "score": 0.2481,
    "source": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md",
    "by": "independent",
    "note": "Committed README v2 benchmark Averaged column /100; not v2.1 and no per-style duplication. HPS v2 Appendix D: 256×256 default configuration, exact author-linked checkpoint Google Drive file 1tMD6MWydRDMaaM7iTOKsUK-Wv2YNDRRt.",
    "source_id": "visual-hpsv2-readme"
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "evalcrafter2023-visual",
    "score": 0.5309,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "evalcrafter2023-alignment",
    "score": 0.5446,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "evalcrafter2023-motion",
    "score": 0.5246999999999999,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "evalcrafter2023-temporal",
    "score": 0.578,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "zeroscope-v2-xl",
    "bench": "evalcrafter2023-visual",
    "score": 0.5341,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Table 1 June 2023 ZeroScope, 1024×576 output. Two-stage generation using 576w base then XL video-to-video upscaler. Separate from zeroscope-v2-576w. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "zeroscope-v2-xl",
    "bench": "evalcrafter2023-alignment",
    "score": 0.5121,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Table 1 June 2023 ZeroScope, 1024×576 output. Two-stage generation using 576w base then XL video-to-video upscaler. Separate from zeroscope-v2-576w. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "zeroscope-v2-xl",
    "bench": "evalcrafter2023-motion",
    "score": 0.5361,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Table 1 June 2023 ZeroScope, 1024×576 output. Two-stage generation using 576w base then XL video-to-video upscaler. Separate from zeroscope-v2-576w. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "zeroscope-v2-xl",
    "bench": "evalcrafter2023-temporal",
    "score": 0.5891,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Table 1 June 2023 ZeroScope, 1024×576 output. Two-stage generation using 576w base then XL video-to-video upscaler. Separate from zeroscope-v2-576w. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "floor33-pictures-2023-08",
    "bench": "evalcrafter2023-visual",
    "score": 0.5878,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Historical hosted model/API evaluated in EvalCrafter. No current API endpoint verified. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "floor33-pictures-2023-08",
    "bench": "evalcrafter2023-alignment",
    "score": 0.6132,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Historical hosted model/API evaluated in EvalCrafter. No current API endpoint verified. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "floor33-pictures-2023-08",
    "bench": "evalcrafter2023-motion",
    "score": 0.4916,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Historical hosted model/API evaluated in EvalCrafter. No current API endpoint verified. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "floor33-pictures-2023-08",
    "bench": "evalcrafter2023-temporal",
    "score": 0.5024000000000001,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Historical hosted model/API evaluated in EvalCrafter. No current API endpoint verified. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "pikalab-2023-09",
    "bench": "evalcrafter2023-visual",
    "score": 0.6077,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Pre-Pika 1.0 historical service (Table 1); not Pika 1.0 or current Pika. No current run option for this snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "pikalab-2023-09",
    "bench": "evalcrafter2023-alignment",
    "score": 0.5579999999999999,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Pre-Pika 1.0 historical service (Table 1); not Pika 1.0 or current Pika. No current run option for this snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "pikalab-2023-09",
    "bench": "evalcrafter2023-motion",
    "score": 0.5577000000000001,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Pre-Pika 1.0 historical service (Table 1); not Pika 1.0 or current Pika. No current run option for this snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "pikalab-2023-09",
    "bench": "evalcrafter2023-temporal",
    "score": 0.6541,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Pre-Pika 1.0 historical service (Table 1); not Pika 1.0 or current Pika. No current run option for this snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "runway-gen-2",
    "bench": "evalcrafter2023-visual",
    "score": 0.6251,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "runway-gen-2",
    "bench": "evalcrafter2023-alignment",
    "score": 0.6098,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "runway-gen-2",
    "bench": "evalcrafter2023-motion",
    "score": 0.5643,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "runway-gen-2",
    "bench": "evalcrafter2023-temporal",
    "score": 0.6441,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "videocrafter1-t2v",
    "bench": "evalcrafter2023-visual",
    "score": 0.6085,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. October 2023 VideoCrafter1 text-to-video, 1024×576, 16 frames; distinct from early VideoCrafter/LVDM and VideoCrafter2. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "videocrafter1-t2v",
    "bench": "evalcrafter2023-alignment",
    "score": 0.6195,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. October 2023 VideoCrafter1 text-to-video, 1024×576, 16 frames; distinct from early VideoCrafter/LVDM and VideoCrafter2. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "videocrafter1-t2v",
    "bench": "evalcrafter2023-motion",
    "score": 0.5307999999999999,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. October 2023 VideoCrafter1 text-to-video, 1024×576, 16 frames; distinct from early VideoCrafter/LVDM and VideoCrafter2. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "videocrafter1-t2v",
    "bench": "evalcrafter2023-temporal",
    "score": 0.5589,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. October 2023 VideoCrafter1 text-to-video, 1024×576, 16 frames; distinct from early VideoCrafter/LVDM and VideoCrafter2. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "show-1",
    "bench": "evalcrafter2023-visual",
    "score": 0.5219,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Original Show-1 multi-stage text-to-video pipeline, 576×320, 8 FPS; not a single component checkpoint. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "show-1",
    "bench": "evalcrafter2023-alignment",
    "score": 0.6207,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Original Show-1 multi-stage text-to-video pipeline, 576×320, 8 FPS; not a single component checkpoint. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "show-1",
    "bench": "evalcrafter2023-motion",
    "score": 0.5374,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Original Show-1 multi-stage text-to-video pipeline, 576×320, 8 FPS; not a single component checkpoint. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "show-1",
    "bench": "evalcrafter2023-temporal",
    "score": 0.6083,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Original Show-1 multi-stage text-to-video pipeline, 576×320, 8 FPS; not a single component checkpoint. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "hotshot-xl",
    "bench": "evalcrafter2023-visual",
    "score": 0.6038,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, visual quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "hotshot-xl",
    "bench": "evalcrafter2023-alignment",
    "score": 0.6152000000000001,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, text-video alignment. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "hotshot-xl",
    "bench": "evalcrafter2023-motion",
    "score": 0.5297999999999999,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, motion quality. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "hotshot-xl",
    "bench": "evalcrafter2023-temporal",
    "score": 0.5996,
    "source": "https://arxiv.org/html/2310.11440v3",
    "by": "independent",
    "note": "Table 2, temporal consistency. Exact 2023 generation settings in Table 1. Gen2 is the September 2023 service snapshot. Human-preference-aligned automatic composite, not raw human MOS; printed score /100.",
    "source_id": "visual-evalcrafter-2023"
  },
  {
    "model": "nvidia-tacotron2-waveglow",
    "bench": "fastpitch2020-ljspeech-mos",
    "score": 0.7365,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md",
    "by": "lab",
    "note": "Committed README MOS table: 3.946 ± 0.134 (95% CI); grapheme inputs and WaveGlow (FastPitch paper Table 1). Store (MOS−1)/4.",
    "source_id": "speech-nvidia-fastpitch-readme"
  },
  {
    "model": "fastpitch-1-0-waveglow",
    "bench": "fastpitch2020-ljspeech-mos",
    "score": 0.77,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md",
    "by": "lab",
    "note": "Committed README MOS table: 4.080 ± 0.133 (95% CI); grapheme inputs and WaveGlow (FastPitch paper Table 1). Store (MOS−1)/4.",
    "source_id": "speech-nvidia-fastpitch-readme"
  },
  {
    "model": "fastpitch-1-0-waveglow",
    "bench": "fastpitch2021-pairwise-preference",
    "score": 0.435,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md",
    "by": "lab",
    "note": "Committed README Average preference table, ±0.068. Pairwise preference between versions 1.0 and 1.1, not human MOS. Stored unchanged.",
    "source_id": "speech-nvidia-fastpitch-readme"
  },
  {
    "model": "fastpitch-1-1-waveglow",
    "bench": "fastpitch2021-pairwise-preference",
    "score": 0.565,
    "source": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md",
    "by": "lab",
    "note": "Committed README Average preference table, ±0.068. Pairwise preference between versions 1.0 and 1.1, not human MOS. Stored unchanged.",
    "source_id": "speech-nvidia-fastpitch-readme"
  },
  {
    "model": "grad-tts-10-hifigan",
    "bench": "gradtts2021-ljspeech-mos",
    "score": 0.845,
    "source": "https://proceedings.mlr.press/v139/popov21a/popov21a.pdf",
    "by": "lab",
    "note": "Table 2 MOS 4.38 ± 0.06 (95% CI), 40 sentences × 10 raters; HiFi-GAN vocoder. Store (MOS−1)/4. Other iteration settings and unspecified baseline checkpoints not pooled.",
    "source_id": "speech-gradtts-icml2021"
  },
  {
    "model": "fastspeech-xcmyz-gradtts-study",
    "bench": "gradtts2021-ljspeech-mos",
    "score": 0.67,
    "source": "https://proceedings.mlr.press/v139/popov21a/popov21a.pdf",
    "by": "independent",
    "note": "Table 2 MOS 3.68 ± 0.09 (95% CI), 40 sentences × 10 raters; HiFi-GAN vocoder. Store (MOS−1)/4. Other iteration settings and unspecified baseline checkpoints not pooled.",
    "source_id": "speech-gradtts-icml2021"
  },
  {
    "model": "melotts-english",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "MeloTTS† Zhao et al. (2023)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.7,
    "raw_value": 3.8,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'MeloTTS† Zhao et al. (2023)' (checkpoint link in Table 5: https://huggingface.co/myshell-ai/MeloTTS-English). Fixed-speaker system; speaker id not stated. Column UTMOS(↑)."
  },
  {
    "model": "parler-tts-mini-v1",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "Parler-TTS† Lacombe et al. (2024)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.7075,
    "raw_value": 3.83,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'Parler-TTS† Lacombe et al. (2024)' (checkpoint link in Table 5: https://huggingface.co/parler-tts/parler-tts-mini-v1). Fixed-speaker system; case-recovery preprocessing applied (Appendix C.2). Column UTMOS(↑)."
  },
  {
    "model": "cosyvoice-300m",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "CosyVoice† Du et al. (2024)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.7875,
    "raw_value": 4.15,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'CosyVoice† Du et al. (2024)' (checkpoint link in Table 5: https://huggingface.co/model-scope/CosyVoice-300M). Marked fixed-speaker (†) although SPK-SIM 0.51 is reported; case-recovery preprocessing applied. Column UTMOS(↑)."
  },
  {
    "model": "vall-e-x-plachtaa",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "VallE-X Zhang et al. (2023c)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.595,
    "raw_value": 3.38,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'VallE-X Zhang et al. (2023c)' (checkpoint link in Table 5: https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt). Zero-shot; the Plachtaa community checkpoint. Column UTMOS(↑)."
  },
  {
    "model": "amphion-valle-v2",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "Vall-E 2 Chen et al. (2024b)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.6625,
    "raw_value": 3.65,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'Vall-E 2 Chen et al. (2024b)' (checkpoint link in Table 5: https://huggingface.co/amphion/valle). Labelled with the Microsoft VALL-E 2 citation but the linked weights are Amphion's reproduction (amphion/valle). Column UTMOS(↑)."
  },
  {
    "model": "amphion-naturalspeech2",
    "source": "https://arxiv.org/html/2412.17667v2",
    "source_id": "speech-versa-toolkit-ccby",
    "by": "independent",
    "date": "2025-03-27",
    "date_basis": "arXiv v2 date (2025-03-27); generation dates not stated",
    "engine_label": "NaturalSpeech2 Shen et al. (2023)",
    "bench": "versa-librispeech-test-clean-utmos",
    "score": 0.3375,
    "raw_value": 2.35,
    "raw_unit": "UTMOS predicted MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "note": "VERSA (arXiv:2412.17667v2) Appendix C.2 Table 6: LibriSpeech test-clean texts, each system run with its developer's open-source pipeline without tuning; zero-shot systems prompted with a random segment of the target speaker. Row 'NaturalSpeech2 Shen et al. (2023)' (checkpoint link in Table 5: https://huggingface.co/amphion/naturalspeech2_libritts). Zero-shot; Amphion's LibriTTS reproduction. Column UTMOS(↑)."
  },
  {
    "model": "neutts-air",
    "bench": "codecmos-accent-vctk-clone-nat-mos",
    "score": 0.765,
    "raw_value": 4.06,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'NeuTTS Air [ 40 ]': S-NAT 4.060 ± 0.070, S-SPK-SIM 3.271 ± 0.093, S-ACC-SIM 3.555 ± 0.093, O-WER 0.759, O-SPK-SIM 0.494, O-UTMOS 4.303. This column: 4.060 ± 0.070 (95% CI). Identity: reference [40] is the neuphonic/neutts GitHub repository; the backbone file used is not stated.",
    "engine_label": "NeuTTS Air",
    "sample_count": 160,
    "ci95_raw": [
      3.99,
      4.13
    ]
  },
  {
    "model": "neutts-air",
    "bench": "codecmos-accent-vctk-clone-spk-mos",
    "score": 0.56775,
    "raw_value": 3.271,
    "raw_unit": "human MOS (1-5)",
    "scale_offset": 1,
    "scale_divisor": 4,
    "source": "https://arxiv.org/html/2603.14328v1",
    "source_id": "speech-codecmos-accent-ccby",
    "by": "independent",
    "date": "2026-03-15",
    "date_basis": "arXiv v1 submission date (2026-03-15); listening-test dates not stated",
    "note": "Table 1 row 'NeuTTS Air [ 40 ]': S-NAT 4.060 ± 0.070, S-SPK-SIM 3.271 ± 0.093, S-ACC-SIM 3.555 ± 0.093, O-WER 0.759, O-SPK-SIM 0.494, O-UTMOS 4.303. This column: 3.271 ± 0.093 (95% CI). Identity: reference [40] is the neuphonic/neutts GitHub repository; the backbone file used is not stated.",
    "engine_label": "NeuTTS Air",
    "sample_count": 160,
    "ci95_raw": [
      3.178,
      3.364
    ]
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "forte",
    "score": 0.703,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'FORTE'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Card column label is 'Gemini 3.1 Pro'; mapped to the catalogue's only Gemini 3.1 Pro identity (gemini-3.1-pro-preview).",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "forte",
    "score": 0.778,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'FORTE'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "forte",
    "score": 0.732,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'FORTE'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "forte",
    "score": 0.776,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'FORTE'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "forte",
    "score": 0.772,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'FORTE'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "rwsearch",
    "score": 0.763,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'RWSearch'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Card column label is 'Gemini 3.1 Pro'; mapped to the catalogue's only Gemini 3.1 Pro identity (gemini-3.1-pro-preview).",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "rwsearch",
    "score": 0.853,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'RWSearch'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "rwsearch",
    "score": 0.813,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'RWSearch'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "rwsearch",
    "score": 0.793,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'RWSearch'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "rwsearch",
    "score": 0.773,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'RWSearch'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "ifeval-unspecified",
    "score": 0.961,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IFEval'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Card column label is 'Gemini 3.1 Pro'; mapped to the catalogue's only Gemini 3.1 Pro identity (gemini-3.1-pro-preview).",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "ifeval-unspecified",
    "score": 0.95,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IFEval'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "ifeval-unspecified",
    "score": 0.922,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IFEval'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "ifeval-unspecified",
    "score": 0.887,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IFEval'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "ifeval-unspecified",
    "score": 0.86,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IFEval'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "writingbench",
    "score": 0.837,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'Writing Bench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Card column label is 'Gemini 3.1 Pro'; mapped to the catalogue's only Gemini 3.1 Pro identity (gemini-3.1-pro-preview).",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "writingbench",
    "score": 0.847,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'Writing Bench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "writingbench",
    "score": 0.853,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'Writing Bench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "writingbench",
    "score": 0.852,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'Writing Bench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "imo-answerbench",
    "score": 0.9,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IMO-AnswerBench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Card column label is 'Gemini 3.1 Pro'; mapped to the catalogue's only Gemini 3.1 Pro identity (gemini-3.1-pro-preview).",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "gpt-5-5",
    "bench": "imo-answerbench",
    "score": 0.795,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IMO-AnswerBench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "imo-answerbench",
    "score": 0.818,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'IMO-AnswerBench'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "gpqa-diamond",
    "score": 0.924,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-07-08",
    "note": "LongCat-2.0 card (pinned 834bf5f, commit dated 2026-07-08), 'Evaluation Results' table, row 'GPQA-diamond'. Card: \"Unless noted with `*`, all scores are measured in-house under a unified harness.\" Unstarred cell = Meituan's own run of this competitor; starred cells (cited from other reports) are excluded. Competitor reasoning/thinking effort, API version and sampling are not stated. Only unstarred competitor cell in this row, so this series has 2 models (LongCat + Opus 4.8) and will not count on its own; recorded for completeness.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "qwen3-8b",
    "bench": "aime-2024",
    "score": 0.779,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-8B row, column \"AIME24\". 16 responses/query, 65536 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "aime-2025",
    "score": 0.658,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-8B row, column \"AIME25\". 16 responses/query, 65536 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.41,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-8B row, column \"HMMT25\". 16 responses/query, 65536 max tokens; edition not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "math-500",
    "score": 0.974,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-8B row, column \"MATH500\". 1 response/query, 32768 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.53,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), Qwen3-8B row, column \"LCB v6\". 3 responses/query, 65536 max tokens; date window not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "scicode-main",
    "score": 0.067,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), Qwen3-8B row, column \"SciCode sub / main\". main-problem value only (subproblem value not imported); 1 response, 65536 max tokens, averaged over 3 repeats, InspectAI. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "gpqa-diamond",
    "score": 0.612,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-8B row, column \"GPQA-Dimond\". 5 responses/query, 32768 max tokens; AA prompts/answer-extraction regex (protocol only). Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "mmlu-pro",
    "score": 0.635,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-8B row, column \"MMLU-Pro\". 3 responses/query, 32768 max tokens; AA prompts/regex (protocol only). Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "hle",
    "score": 0.042,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-8B row, column \"HLE\". 2,158 text-only problems, 1 response, 65536 max tokens; GPT-4o equality checker. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "ifbench-unspecified",
    "score": 0.353,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-8B row, column \"IFBench\". 1 response/query, 32768 max tokens; aggregation not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "aime-2024",
    "score": 0.833,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"AIME24\". 16 responses/query, 65536 max tokens. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "aime-2025",
    "score": 0.758,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"AIME25\". 16 responses/query, 65536 max tokens. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.543,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"HMMT25\". 16 responses/query, 65536 max tokens; edition not stated. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "math-500",
    "score": 0.968,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"MATH500\". 1 response/query, 32768 max tokens. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.572,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"LCB v6\". 3 responses/query, 65536 max tokens; date window not stated. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "scicode-main",
    "score": 0.026,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), DeepSeek-R1-0528-Qwen3-8B row, column \"SciCode sub / main\". main-problem value only (subproblem value not imported); 1 response, 65536 max tokens, averaged over 3 repeats, InspectAI. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "gpqa-diamond",
    "score": 0.614,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), DeepSeek-R1-0528-Qwen3-8B row, column \"GPQA-Dimond\". 5 responses/query, 32768 max tokens; AA prompts/answer-extraction regex (protocol only). Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "mmlu-pro",
    "score": 0.691,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), DeepSeek-R1-0528-Qwen3-8B row, column \"MMLU-Pro\". 3 responses/query, 32768 max tokens; AA prompts/regex (protocol only). Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "hle",
    "score": 0.056,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), DeepSeek-R1-0528-Qwen3-8B row, column \"HLE\". 2,158 text-only problems, 1 response, 65536 max tokens; GPT-4o equality checker. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "ifbench-unspecified",
    "score": 0.292,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), DeepSeek-R1-0528-Qwen3-8B row, column \"IFBench\". 1 response/query, 32768 max tokens; aggregation not stated. Reasoning model; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2024",
    "score": 0.833,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), GPT-OSS-20B row, column \"AIME24\". 16 responses/query, 65536 max tokens. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2025",
    "score": 0.844,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), GPT-OSS-20B row, column \"AIME25\". 16 responses/query, 65536 max tokens. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.648,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), GPT-OSS-20B row, column \"HMMT25\". 16 responses/query, 65536 max tokens; edition not stated. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "math-500",
    "score": 0.948,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), GPT-OSS-20B row, column \"MATH500\". 1 response/query, 32768 max tokens. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.72,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), GPT-OSS-20B row, column \"LCB v6\". 3 responses/query, 65536 max tokens; date window not stated. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "scicode-main",
    "score": 0.062,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), GPT-OSS-20B row, column \"SciCode sub / main\". main-problem value only (subproblem value not imported); 1 response, 65536 max tokens, averaged over 3 repeats, InspectAI. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "gpqa-diamond",
    "score": 0.612,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), GPT-OSS-20B row, column \"GPQA-Dimond\". 5 responses/query, 32768 max tokens; AA prompts/answer-extraction regex (protocol only). Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmlu-pro",
    "score": 0.756,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), GPT-OSS-20B row, column \"MMLU-Pro\". 3 responses/query, 32768 max tokens; AA prompts/regex (protocol only). Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "hle",
    "score": 0.098,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), GPT-OSS-20B row, column \"HLE\". 2,158 text-only problems, 1 response, 65536 max tokens; GPT-4o equality checker. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "ifbench-unspecified",
    "score": 0.694,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), GPT-OSS-20B row, column \"IFBench\". 1 response/query, 32768 max tokens; aggregation not stated. Reasoning effort not stated; each baseline's recommended evaluation configuration. TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2024",
    "score": 0.794,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-32B row, column \"AIME24\". 16 responses/query, 65536 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2025",
    "score": 0.71,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-32B row, column \"AIME25\". 16 responses/query, 65536 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.498,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-32B row, column \"HMMT25\". 16 responses/query, 65536 max tokens; edition not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "math-500",
    "score": 0.968,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 4 (Mathematical reasoning evaluations), Qwen3-32B row, column \"MATH500\". 1 response/query, 32768 max tokens. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.61,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), Qwen3-32B row, column \"LCB v6\". 3 responses/query, 65536 max tokens; date window not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "scicode-main",
    "score": 0.092,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 5 (Code generation with reasoning evaluations), Qwen3-32B row, column \"SciCode sub / main\". main-problem value only (subproblem value not imported); 1 response, 65536 max tokens, averaged over 3 repeats, InspectAI. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "gpqa-diamond",
    "score": 0.673,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-32B row, column \"GPQA-Dimond\". 5 responses/query, 32768 max tokens; AA prompts/answer-extraction regex (protocol only). Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-pro",
    "score": 0.739,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-32B row, column \"MMLU-Pro\". 3 responses/query, 32768 max tokens; AA prompts/regex (protocol only). Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "hle",
    "score": 0.083,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-32B row, column \"HLE\". 2,158 text-only problems, 1 response, 65536 max tokens; GPT-4o equality checker. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "ifbench-unspecified",
    "score": 0.354,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Table 6 (General domains reasoning), Qwen3-32B row, column \"IFBench\". 1 response/query, 32768 max tokens; aggregation not stated. Thinking toggle not stated; paper evaluates baselines as reasoning models with each baseline's recommended configuration (Qwen3 default is thinking enabled). TII authors' own baseline run (Section 4.1: \"To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations\"). Regular pass@1 sampling, not DeepConf (Table 7).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "imo-answerbench",
    "score": 0.704,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column (reasoning model, NVIDIA run). IMO-AnswerBench: LLM-judge (GPT-OSS-120B, AnswerAutoGrader prompt) score, avg@16, Cascade-2 run at 256K-token thinking budget; baselines \"evaluate them with the recommended settings if the official numbers are unavailable\" (Appendix A.1.1). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "imo-answerbench",
    "score": 0.772,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Super 120B-A12B column (NVIDIA run). IMO-AnswerBench: LLM-judge (GPT-OSS-120B, AnswerAutoGrader prompt) score, avg@16, Cascade-2 run at 256K-token thinking budget; baselines \"evaluate them with the recommended settings if the official numbers are unavailable\" (Appendix A.1.1). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "imo-answerbench",
    "score": 0.748,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking, recommended settings, NVIDIA run). IMO-AnswerBench: LLM-judge (GPT-OSS-120B, AnswerAutoGrader prompt) score, avg@16, Cascade-2 run at 256K-token thinking budget; baselines \"evaluate them with the recommended settings if the official numbers are unavailable\" (Appendix A.1.1). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "aime-2026",
    "score": 0.909,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column, leading no-tool value. AIME 2026 (30 problems), avg@64; Cascade-2 at 131K-token thinking budget, temperature 1.0, top-p 1.0, no tools (bracketed † TIR value not used); baselines run by NVIDIA with recommended settings. Target row needed so this report's AIME 2026 series exists (existing aime-2026 row is from the HF card, a different source).",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "aime-2026",
    "score": 0.899,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column. AIME 2026 (30 problems), avg@64; Cascade-2 at 131K-token thinking budget, temperature 1.0, top-p 1.0, no tools (bracketed † TIR value not used); baselines run by NVIDIA with recommended settings. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "aime-2026",
    "score": 0.898,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Super 120B-A12B column. AIME 2026 (30 problems), avg@64; Cascade-2 at 131K-token thinking budget, temperature 1.0, top-p 1.0, no tools (bracketed † TIR value not used); baselines run by NVIDIA with recommended settings. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "aime-2026",
    "score": 0.911,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking). AIME 2026 (30 problems), avg@64; Cascade-2 at 131K-token thinking budget, temperature 1.0, top-p 1.0, no tools (bracketed † TIR value not used); baselines run by NVIDIA with recommended settings. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.87,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column, leading no-tool value. Easy split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.276,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column, leading no-tool value. Med split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.545,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column. Easy split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.035,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column. Med split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.817,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Super 120B-A12B column. Easy split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.232,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Super 120B-A12B column. Med split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.811,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking). Easy split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.178,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking). Med split. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode (Appendix A); Cascade-2 at 128K-token thinking budget, temperature 1.0, top-p 0.95, no tools; baselines \"with their recommended inference configurations, ensuring a thinking budget of at least 128K tokens\". Same numbers repeated in Table 6. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "cl-bench",
    "score": 0.122,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column. CL-Bench (1,899 samples, 500 contexts), thinking mode, pass@1 from a single generation (avg@1) (Appendix A).",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "cl-bench",
    "score": 0.12,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column. CL-Bench (1,899 samples, 500 contexts), thinking mode, pass@1 from a single generation (avg@1) (Appendix A). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "cl-bench",
    "score": 0.155,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking). CL-Bench (1,899 samples, 500 contexts), thinking mode, pass@1 from a single generation (avg@1) (Appendix A). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "aime-2025",
    "score": 0.924,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column, leading no-tool value; avg@64, 131K thinking budget. Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "aime-2025",
    "score": 0.919,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column (thinking); avg@64. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded. Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "hmmt-feb-2025",
    "score": 0.946,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column; HMMT Feb 2025, avg@64, 131K thinking budget. Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "hmmt-feb-2025",
    "score": 0.846,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-3-Nano 30B-A3B column; HMMT Feb 2025. Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded. Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.835,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Nemotron-Cascade-2 30B-A3B column; ArenaHard v2 (Avg.) aggregate of Hard Prompt and Creative Writing (components not imported). Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.654,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "independent",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report v2, Table 1 \"Main results\", Qwen3.5 35B-A3B column; ArenaHard v2 (Avg.). Cell marked ‡ = baseline evaluated by NVIDIA with the recommended settings (caption: \"‡ For the baseline models, we use official numbers when available, otherwise evaluate them using the recommended settings.\"). Cross-check: every unmarked Qwen3.5-35B-A3B cell (MMLU-Pro 85.3, GPQA-Diamond 84.2, HMMT Feb25 89.0, LCB v6 74.6, IFBench 70.2 ...) is a number Qwen publishes in its own card, while every ‡ cell is a benchmark Qwen does not report, so ‡ marks NVIDIA's own runs; unmarked baseline cells are copied official numbers and are excluded. Only one ‡ baseline on this row, so this report series has 2 models and does not calibrate on its own; recorded because the cells are usable.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.593,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2), Table 9 \"Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models\", LiveCodeBench Pro 25Q2 columns: 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode. Appendix B: \"We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.\" Row \"NVIDIA-Nemotron-Nano-9B-v2\" (NVIDIA's own released model, reasoning on); no official LCB Pro number exists for it elsewhere, so this is NVIDIA's own run."
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.048,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2), Table 9 \"Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models\", LiveCodeBench Pro 25Q2 columns: 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode. Appendix B: \"We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.\" Row \"NVIDIA-Nemotron-Nano-9B-v2\" (reasoning on)."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu",
    "score": 0.624,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmlu",
    "score": 0.698,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu",
    "score": 0.808,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "mmlu",
    "score": 0.798,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmlu",
    "score": 0.762,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmlu",
    "score": 0.767,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-pro",
    "score": 0.352,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmlu-pro",
    "score": 0.495,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.669,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "mmlu-pro",
    "score": 0.639,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmlu-pro",
    "score": 0.558,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmlu-pro",
    "score": 0.571,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "agieval-marco-unspecified",
    "score": 0.396,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "agieval-marco-unspecified",
    "score": 0.447,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "agieval-marco-unspecified",
    "score": 0.517,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "agieval-marco-unspecified",
    "score": 0.524,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "agieval-marco-unspecified",
    "score": 0.436,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "agieval-marco-unspecified",
    "score": 0.447,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. AGIEval subtests not listed. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-diamond",
    "score": 0.275,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "gpqa-diamond",
    "score": 0.316,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "gpqa-diamond",
    "score": 0.508,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "gpqa-diamond",
    "score": 0.448,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "gpqa-diamond",
    "score": 0.352,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "gpqa-diamond",
    "score": 0.386,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gsm8k",
    "score": 0.779,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "gsm8k",
    "score": 0.79,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "gsm8k",
    "score": 0.886,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "gsm8k",
    "score": 0.895,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "gsm8k",
    "score": 0.897,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "gsm8k",
    "score": 0.839,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "math-subset-unspecified",
    "score": 0.706,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "math-subset-unspecified",
    "score": 0.702,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "math-subset-unspecified",
    "score": 0.934,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.862,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "math-subset-unspecified",
    "score": 0.838,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "math-subset-unspecified",
    "score": 0.757,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. MATH subset (full test vs MATH-500) not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "global-mmlu-marco",
    "score": 0.463,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "global-mmlu-marco",
    "score": 0.384,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "global-mmlu-marco",
    "score": 0.702,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "global-mmlu-marco",
    "score": 0.554,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "global-mmlu-marco",
    "score": 0.692,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "global-mmlu-marco",
    "score": 0.674,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmmlu-marco",
    "score": 0.49,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmmlu-marco",
    "score": 0.394,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmmlu-marco",
    "score": 0.713,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "mmmlu-marco",
    "score": 0.564,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmmlu-marco",
    "score": 0.694,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmmlu-marco",
    "score": 0.681,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Acc@1, one sampled response per query. Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.286,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.267,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.583,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.433,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.513,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.516,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mgsm-marco",
    "score": 0.636,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "mgsm-marco",
    "score": 0.47,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mgsm-marco",
    "score": 0.844,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "mgsm-marco",
    "score": 0.687,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "mgsm-marco",
    "score": 0.84,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "mgsm-marco",
    "score": 0.757,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "polymath-marco",
    "score": 0.234,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 1 \"Qwen3 1.7B Instruct\"; Section 4.3.1: \"Qwen3-1.7B-Instruct (i.e., non-thinking)\", i.e. Qwen/Qwen3-1.7B with thinking disabled. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Non-thinking mode. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-3b",
    "bench": "polymath-marco",
    "score": 0.163,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 3 \"Ministral3 3B Instruct\" (3.84B params); Section 4.3.1 names Ministral-3-3B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "polymath-marco",
    "score": 0.472,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 7 \"Qwen3 4B Instruct\"; Section 4.3.1 names Qwen3-4B-Instruct-2507. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "ministral-3-8b",
    "bench": "polymath-marco",
    "score": 0.264,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 8 \"Ministral3 8B Instruct\" (8.8B params); Section 4.3.1 names Ministral-3-8B-Instruct-2512. Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "gemma-3-12b",
    "bench": "polymath-marco",
    "score": 0.355,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 9 \"Gemma3 12B Instruct\"; Section 4.3.1 names Gemma3-12B-Instruct (google/gemma-3-12b-it). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "polymath-marco",
    "score": 0.289,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-04-28",
    "note": "Marco-MoE report (arXiv 2604.25578v1) Table 11; the Marco authors ran every baseline themselves (Section 4.3.1: all models are evaluated with the Light-Eval framework). Column 10 \"Granite4 Small Instruct\", table lists 9B activated / 32B total; Section 4.3.1 names Granite4-Small, cited as IBM Granite 4.0 language models, which matches ibm-granite/granite-4.0-h-small (the only Granite 4.0 \"Small\" release). Temperature 1.0, top_p 1.0, max response 8,192 tokens; Avg@8, mean accuracy over 8 sampled responses per query (not pass@8 or majority vote). Shot count not stated. Same table and series as the existing lfm2-24b-a2b rows."
  },
  {
    "model": "aya-23-35b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8352,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-23-35B', instruction-tuned (📝), default setting; EuroEval version 15.10.0.dev0."
  },
  {
    "model": "aya-23-35b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8176,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-23-35B', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "aya-23-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7129,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-23-8B', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "aya-expanse-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8814,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-expanse-32b', instruction-tuned (📝), default setting; EuroEval version 15.4.1.dev0."
  },
  {
    "model": "aya-expanse-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8674,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-expanse-32b', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "aya-expanse-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7666,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/aya-expanse-8b', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "cogito-v1-preview-llama-3b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6491,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-3B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "cogito-v1-preview-llama-3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.733,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-3B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "cogito-v1-preview-llama-8b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.786,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-8B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "cogito-v1-preview-llama-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8186,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-8B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "command-r-03-2024",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8426,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/c4ai-command-r-v01', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "command-r-08-2024",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8816,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'CohereForAI/c4ai-command-r-08-2024', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6896,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-0528-Qwen3-8B', reasoning model (🤔), default setting; EuroEval version 15.9.1."
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8305,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-0528-Qwen3-8B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3975,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Llama-70B', reasoning model (🤔), default setting; EuroEval version 15.9.2.dev0."
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5368,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Llama-8B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.783,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Llama-8B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3214,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7261,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5377,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7949,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', reasoning model (🤔), default setting; EuroEval version 15.10.0.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7445,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.4899,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5387,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.7969,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LGAI-EXAONE/EXAONE-4.0-32B (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.866,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LGAI-EXAONE/EXAONE-4.0-32B (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-2-27b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8297,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-27b-it', instruction-tuned (📝), default setting; EuroEval version 15.6.1."
  },
  {
    "model": "gemma-2-27b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8818,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-27b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-2-2b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5394,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-2b-it', instruction-tuned (📝), default setting; EuroEval version 15.4.1."
  },
  {
    "model": "gemma-2-2b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6998,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-2b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-2-9b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7825,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-9b-it', instruction-tuned (📝), default setting; EuroEval version 15.6.1."
  },
  {
    "model": "gemma-2-9b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8434,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2-9b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-3-12b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8091,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-12b-it', instruction-tuned (📝), default setting; EuroEval version 15.5.0."
  },
  {
    "model": "gemma-3-12b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.85,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-12b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-3-1b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.4371,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-1b-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "gemma-3-27b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.821,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-27b-it', instruction-tuned (📝), default setting; EuroEval version 15.9.1."
  },
  {
    "model": "gemma-3-27b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9211,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-27b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-3-4b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5787,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-4b-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "gemma-3-4b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.777,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-4b-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6643,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3n-E2B-it', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0."
  },
  {
    "model": "gemma-3n-e2b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7941,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3n-E2B-it', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0."
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7131,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3n-E4B-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "gemma-3n-e4b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8018,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3n-E4B-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.8227,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-26B-A4B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9344,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-26B-A4B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-31b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.8906,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-31B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-31b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9516,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-31B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-e2b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.5363,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-E2B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-e2b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.6965,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-E2B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-e4b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.6996,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-E4B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-4-e4b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.8152,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-4-E4B-it (val)', instruction-tuned (📝), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gemma-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6404,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-7b-it', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "gemma-7b-base",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3352,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-7b', base model (🧠), default setting; EuroEval version 15.6.1."
  },
  {
    "model": "gemma-7b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.849,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-7b', base model (🧠), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "glm-4-5-air",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8042,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'zai-org/GLM-4.5-Air#thinking', reasoning model (🤔), thinking mode (#thinking); EuroEval version 16.2.2.dev0."
  },
  {
    "model": "glm-4-5-air",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9584,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'zai-org/GLM-4.5-Air#thinking', reasoning model (🤔), thinking mode (#thinking); EuroEval version 16.2.2.dev0."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.748,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'zai-org/GLM-4.7-Flash (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "glm-4-7-flash",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9406,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'zai-org/GLM-4.7-Flash (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6773,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai/gpt-oss-20b#high', reasoning model (🤔), reasoning effort high (#high); EuroEval version 16.12.0.dev0."
  },
  {
    "model": "gpt-oss-20b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7271,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai/gpt-oss-20b#high', reasoning model (🤔), reasoning effort high (#high); EuroEval version 16.12.0.dev0."
  },
  {
    "model": "granite-3-0-2b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7133,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3.0-2b-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "granite-3-0-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8064,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3.0-8b-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3344,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3b-code-base-2k', base model (🧠), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3547,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3b-code-instruct-2k', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8479,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-small', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0."
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8648,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-small', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0."
  },
  {
    "model": "granite-4-0-micro",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7076,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-micro', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0."
  },
  {
    "model": "granite-4-0-micro",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7734,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-micro', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0."
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3447,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-8b-code-base-4k', base model (🧠), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "granite-8b-code-instruct-4k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.4516,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-8b-code-instruct-4k', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3892,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2.5-1.2B-Thinking', reasoning model (🤔), default setting; EuroEval version 16.10.1.dev0."
  },
  {
    "model": "lfm2-5-1-2b-thinking",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7523,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2.5-1.2B-Thinking', reasoning model (🤔), default setting; EuroEval version 16.10.1.dev0."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5764,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-13b-chat-hf', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-2-70b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5898,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-70b-hf', base model (🧠), default setting; EuroEval version 15.3.1."
  },
  {
    "model": "llama-2-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8602,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-70b-hf', base model (🧠), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5746,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-70b-chat-hf', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5434,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-7b-chat-hf', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-1-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9391,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-3.1-70B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-1-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8441,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-3.1-8B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9352,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8389,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Llama-3.1-Tulu-3-8B', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3238,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-3.2-1B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "llama-3-2-3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7697,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-3.2-3B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-3-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9436,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-3.3-70B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9432,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Meta-Llama-3-70B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-3-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8346,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Meta-Llama-3-8B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "llama-4-scout",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8644,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-4-Scout-17B-16E-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0."
  },
  {
    "model": "llama-4-scout",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9105,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-4-Scout-17B-16E-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0."
  },
  {
    "model": "magistral-small-2509",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8942,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Magistral-Small-2509', instruction-tuned (📝), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "magistral-small-2509",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9549,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Magistral-Small-2509', instruction-tuned (📝), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "ministral-3-14b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.793,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-14B-Instruct-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-3-14b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9266,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-14B-Instruct-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.8688,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-14B-Reasoning-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.932,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-14B-Reasoning-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-3-3b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.698,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-3B-Instruct-2512', instruction-tuned (📝), default setting; EuroEval version 16.17.0."
  },
  {
    "model": "ministral-3-3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8148,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-3B-Instruct-2512', instruction-tuned (📝), default setting; EuroEval version 16.17.0."
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7558,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-3B-Reasoning-2512', instruction-tuned (📝), default setting; EuroEval version 16.17.0."
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7809,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-3B-Reasoning-2512', instruction-tuned (📝), default setting; EuroEval version 16.17.0."
  },
  {
    "model": "ministral-3-8b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.7312,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-8B-Instruct-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-3-8b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.8969,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-3-8B-Instruct-2512 (val)', instruction-tuned (📝), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "ministral-8b-2410",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8252,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Ministral-8B-Instruct-2410', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6898,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-7B-Instruct-v0.1', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6541,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-7B-Instruct-v0.2', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8457,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Nemo-Instruct-2407', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-small-3",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9101,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-24B-Instruct-2501', instruction-tuned (📝), default setting; EuroEval version 15.0.0."
  },
  {
    "model": "mistral-small-3",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9434,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-24B-Instruct-2501', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9311,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-3.1-24B-Instruct-2503', instruction-tuned (📝), default setting; EuroEval version 15.9.1.dev0."
  },
  {
    "model": "mistral-small-3-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9439,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-3.1-24B-Instruct-2503', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9042,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-3.2-24B-Instruct-2506', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0."
  },
  {
    "model": "mistral-small-3-2",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9514,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mistral-Small-3.2-24B-Instruct-2506', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0."
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8734,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mixtral-8x7B-Instruct-v0.1', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8715,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'mistralai/Mixtral-8x7B-v0.1', base model (🧠), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.6922,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9301,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.7082,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/Nemotron-Cascade-2-30B-A3B (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "nemotron-cascade-2-30b-a3b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.9328,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/Nemotron-Cascade-2-30B-A3B (val)', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6244,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/NVIDIA-Nemotron-Nano-9B-v2', reasoning model (🤔), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9207,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/NVIDIA-Nemotron-Nano-9B-v2', reasoning model (🤔), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "o3-mini",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.7578,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'o3-mini-2025-01-31 (val)', reasoning model (🤔), default setting; EuroEval version 15.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "olmo-2-13b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6854,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-1124-13B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0.dev0."
  },
  {
    "model": "olmo-2-13b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8303,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-1124-13B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0.dev0."
  },
  {
    "model": "olmo-2-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8062,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-0325-32B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.9.2."
  },
  {
    "model": "olmo-2-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9082,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-0325-32B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "olmo-2-7b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6407,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-1124-7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0."
  },
  {
    "model": "olmo-2-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7926,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/OLMo-2-1124-7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.13.0."
  },
  {
    "model": "olmo-3-1-32b-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7668,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3.1-32B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "olmo-3-1-32b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8455,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3.1-32B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7432,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3.1-32B-Think', reasoning model (🤔), default setting; EuroEval version 16.14.0.dev0."
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8986,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3.1-32B-Think', reasoning model (🤔), default setting; EuroEval version 16.14.0.dev0."
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6125,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3-7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7508,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3-7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0."
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6038,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3-7B-Think', reasoning model (🤔), default setting; EuroEval version 16.11.0.dev0."
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8574,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'allenai/Olmo-3-7B-Think', reasoning model (🤔), default setting; EuroEval version 16.11.0.dev0."
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8018,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-3-mini-128k-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "phi-4",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8438,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/phi-4', instruction-tuned (📝), default setting; EuroEval version 15.9.2."
  },
  {
    "model": "phi-4",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8922,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/phi-4', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.4099,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-4-mini-reasoning', reasoning model (🤔), default setting; EuroEval version 15.9.2."
  },
  {
    "model": "phi-4-mini-reasoning",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5572,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-4-mini-reasoning', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "phi-4-reasoning",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.4871,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-4-reasoning', reasoning model (🤔), default setting; EuroEval version 15.9.2.dev0."
  },
  {
    "model": "phi-4-reasoning",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8352,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-4-reasoning', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7961,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen1.5-14B-Chat', instruction-tuned (📝), default setting; EuroEval version 15.3.1."
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8275,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen1.5-14B-Chat', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen1-5-4b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7283,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen1.5-4B-Chat', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6465,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen2.5-1.5B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.76,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen2.5-1.5B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen2-5-72b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9631,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen2.5-72B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen3-0-6b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3857,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-0.6B', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0."
  },
  {
    "model": "qwen3-0-6b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6186,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-0.6B', reasoning model (🤔), default setting; EuroEval version 17.1.0.dev0."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.463,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-1.7B', instruction-tuned (📝), default setting; EuroEval version 16.1.1.dev0."
  },
  {
    "model": "qwen3-1-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6836,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-1.7B', instruction-tuned (📝), default setting; EuroEval version 16.1.1.dev0."
  },
  {
    "model": "qwen3-14b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7977,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-14B', reasoning model (🤔), default setting; EuroEval version 15.7.2.dev0."
  },
  {
    "model": "qwen3-14b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9352,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-14B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8591,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-235B-A22B', reasoning model (🤔), default setting; EuroEval version 15.12.0.dev0."
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9559,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-235B-A22B', reasoning model (🤔), default setting; EuroEval version 15.12.0.dev0."
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8171,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-30B-A3B', reasoning model (🤔), default setting; EuroEval version 15.8.1."
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.948,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-30B-A3B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8826,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-30B-A3B-Instruct-2507', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0."
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9064,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-30B-A3B-Instruct-2507', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0."
  },
  {
    "model": "qwen3-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8399,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-32B', reasoning model (🤔), default setting; EuroEval version 15.8.2."
  },
  {
    "model": "qwen3-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9408,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-32B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen3-4b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.708,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B', instruction-tuned (📝), default setting; EuroEval version 16.1.1.dev0."
  },
  {
    "model": "qwen3-4b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8242,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B', instruction-tuned (📝), default setting; EuroEval version 16.1.1.dev0."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.773,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B-Instruct-2507', instruction-tuned (📝), default setting; EuroEval version 16.0.0.dev0."
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8166,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B-Instruct-2507', instruction-tuned (📝), default setting; EuroEval version 16.0.0.dev0."
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7403,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B-Thinking-2507', reasoning model (🤔), default setting; EuroEval version 16.14.0.dev0."
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8906,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-4B-Thinking-2507', reasoning model (🤔), default setting; EuroEval version 16.14.0.dev0."
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.3797,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-0.8B (val)', instruction-tuned (📝), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.5625,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-0.8B (val)', instruction-tuned (📝), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-2b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.5621,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-2B (val)', instruction-tuned (📝), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-2b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.7062,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-2B (val)', instruction-tuned (📝), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-4b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.7691,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-4B (val)', reasoning model (🤔), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-4b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.8613,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-4B (val)', reasoning model (🤔), default setting; EuroEval version 16.17.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-9b",
    "bench": "euroeval-en-hellaswag-val-5shot",
    "score": 0.8211,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-9B (val)', reasoning model (🤔), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-5-9b",
    "bench": "euroeval-en-life-in-the-uk-val-5shot",
    "score": 0.898,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), val split (256 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3.5-9B (val)', reasoning model (🤔), default setting; EuroEval version 17.0.0.dev0. EuroEval FAQ: '(val)' runs are validation-split only and 'aren't strictly comparable to the regular test-split scores', hence a separate benchmark id."
  },
  {
    "model": "qwen3-8b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7548,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-8B', reasoning model (🤔), default setting; EuroEval version 15.7.1."
  },
  {
    "model": "qwen3-8b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8977,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-8B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9048,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-Next-80B-A3B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9438,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-Next-80B-A3B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8814,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-Next-80B-A3B-Thinking', reasoning model (🤔), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9592,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen3-Next-80B-A3B-Thinking', reasoning model (🤔), default setting; EuroEval version 16.3.0.dev0."
  },
  {
    "model": "qwq-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8527,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/QwQ-32B', reasoning model (🤔), default setting; EuroEval version 15.6.1.dev0."
  },
  {
    "model": "qwq-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.932,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/QwQ-32B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "smollm2-1-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5025,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'HuggingFaceTB/SmolLM2-1.7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "smollm3-3b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5895,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'HuggingFaceTB/SmolLM3-3B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "smollm3-3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7568,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'HuggingFaceTB/SmolLM3-3B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8275,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Nexusflow/Starling-LM-7B-beta', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0."
  },
  {
    "model": "grok-voice-transcribe-2",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0079,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'xAI | grok-voice-transcribe-2.0', Pooled WER 0.79% (WER mean 1.12%, transcripts 99.8%). services.py create_xai(): XAISTTService model='grok-voice-transcribe-2.0' (streaming). Added by Pipecat maintainer markbackman in commit 7e438f9 ('Add xAI grok-voice-transcribe-2.0 results', 2026-10-01), scored by the Sonnet 5.5 judge."
  },
  {
    "model": "assemblyai-universal-3-5-pro",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0095,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'AssemblyAI | universal-3-5-pro', Pooled WER 0.95%. services.py create_assemblyai_universal_3_5_pro(): AssemblyAISTTService (streaming) model='universal-3-5-pro'; code comment states the sync variant is 'the same universal-3-5-pro model'. Re-scored with the Sonnet 5.5 judge in commit f78a7a2 (2026-10-01)."
  },
  {
    "model": "deepgram-nova-3",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0137,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Deepgram | nova-3-general', Pooled WER 1.37%. services.py create_deepgram(): DeepgramSTTService (streaming) model='nova-3-general' (Nova-3, general variant). Re-scored with the Sonnet 5.5 judge in commit f78a7a2."
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0295,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'OpenAI | gpt-4o-transcribe', Pooled WER 2.95%. services.py create_openai_realtime(): OpenAIRealtimeSTTService model='gpt-4o-transcribe' (Realtime transcription API). Re-scored with the Sonnet 5.5 judge in commit f78a7a2."
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0511,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Mistral | voxtral-mini-transcribe-realtime-2602', Pooled WER 5.11%. services.py create_mistral(): MistralSTTService model='voxtral-mini-transcribe-realtime-2602' (Mistral-hosted API name of the catalogue model; catalogue api.url is docs.mistral.ai/models/model-cards/voxtral-mini-transcribe-realtime-26-02). Hosted endpoint, delay setting not stated. Re-scored with the Sonnet 5.5 judge in commit f78a7a2."
  },
  {
    "model": "maya1",
    "bench": "dailydialog-tts-whisper-large-wer",
    "score": 0.1122,
    "source": "https://arxiv.org/html/2601.20510v1",
    "by": "independent",
    "date": "2026-01-28",
    "note": "Singh et al. (UncovAI), 'Audio Deepfake Detection in the Age of Advanced Text-to-Speech models', arXiv:2601.20510v1, Table 2 'Quantitative Evaluation of Generated TTS Samples'. Text from DailyDialog transcripts; WER by OpenAI Whisper-large + jiwer against the DailyDialog text. Model described as the 3B Llama-backbone SNAC codec model Maya1 (maya-research/maya1). Dia2 2B from the same table was added on 2026-10-06; this three-model study can now calibrate. No synthetic values were introduced."
  },
  {
    "model": "cogvideo-9b",
    "bench": "fetv-human-video-quality",
    "score": 0.5675,
    "source": "https://arxiv.org/html/2311.01813v3#A6.F20",
    "by": "independent",
    "date": "2023-12-26",
    "note": "CogVideo; FETV manual video quality 3.27 on 1-5 (stored (x-1)/4). App. C.1: official implementation, default hyper-parameters, both-stage Transformers (9.4B total), 33 frames, 480x480. Prompt language handling for this Chinese-trained model not stated."
  },
  {
    "model": "zeroscope-v2-576w",
    "bench": "fetv-human-video-quality",
    "score": 0.655,
    "source": "https://arxiv.org/html/2311.01813v3#A6.F20",
    "by": "independent",
    "date": "2023-12-26",
    "note": "ZeroScope; FETV manual video quality 3.62 on 1-5 (stored (x-1)/4). App. C.4: 'We adopt the ZeroScope-v2-576w model, following the official implementation in HuggingFace', 3 s at 8 fps, 576x320."
  },
  {
    "model": "modelscope-t2v-1-7b",
    "bench": "fetv-human-video-quality",
    "score": 0.7325,
    "source": "https://arxiv.org/html/2311.01813v3#A6.F20",
    "by": "independent",
    "date": "2023-12-26",
    "note": "ModelScopeT2V; FETV manual video quality 3.93 on 1-5 (stored (x-1)/4). App. C.3: 'code and model checkpoints released in ModelScope', 2 s at 8 fps, 256x256 (the ModelScope text-to-video 1.7B release)."
  },
  {
    "model": "gemini-4-argon",
    "bench": "lmarena-text",
    "score": 1525.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gemini-4-argon-high'; 4,932 votes; published 95% interval 1516.4–1534.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gemini-4-argon",
    "bench": "lmarena-webdev",
    "score": 1679.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gemini-4-argon-high'; 2,422 votes; published 95% interval 1666.2–1692.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "lmarena-text",
    "score": 1483.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-6.1-sol-max'; 3,071 votes; published 95% interval 1472.6–1493.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "lmarena-vision",
    "score": 1290.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-6.1-sol-max'; 1,408 votes; published 95% interval 1274.6–1306.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "lmarena-webdev",
    "score": 1757.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'gpt-6.1-sol-max'; 1,620 votes; published 95% interval 1741.0–1774.6. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "lmarena-text",
    "score": 1471,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-sonnet-5.5-xhigh'; 3,145 votes; published 95% interval 1460.6–1481.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "lmarena-vision",
    "score": 1268.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-sonnet-5.5-xhigh'; 1,608 votes; published 95% interval 1253.2–1283.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "lmarena-webdev",
    "score": 1786.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'claude-sonnet-5.5-xhigh'; 1,531 votes; published 95% interval 1768.1–1804.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-5-preview",
    "bench": "lmarena-text",
    "score": 1453.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'Step 5 Preview'; 2,701 votes; published 95% interval 1442.6–1464.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-5-preview",
    "bench": "lmarena-vision",
    "score": 1259.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'Step 5 Preview'; 604 votes; published 95% interval 1235.5–1284.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "dola-seed-2-0-pro",
    "bench": "lmarena-text",
    "score": 1456.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'dola-seed-2.0-pro'; 78,890 votes; published 95% interval 1453.1–1459.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "dola-seed-2-0-pro",
    "bench": "lmarena-vision",
    "score": 1256.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'dola-seed-2.0-pro'; 10,981 votes; published 95% interval 1249.3–1263.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-3-chat-latest-20261002",
    "bench": "lmarena-text",
    "score": 1450,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5.3-chat-latest'; 34,296 votes; published 95% interval 1445.8–1454.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-chat-20261002",
    "bench": "lmarena-text",
    "score": 1426.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'gpt-5-chat'; 31,009 votes; published 95% interval 1422.4–1431.0. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "gpt-5-chat-20261002",
    "bench": "lmarena-vision",
    "score": 1225.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'gpt-5-chat'; 39,910 votes; published 95% interval 1218.6–1232.7. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mercury",
    "bench": "lmarena-text",
    "score": 1305.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mercury'; 1,949 votes; published 95% interval 1291.4–1319.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mercury-2",
    "bench": "lmarena-text",
    "score": 1343.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mercury-2'; 3,197 votes; published 95% interval 1332.8–1353.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "mercury-2",
    "bench": "lmarena-webdev",
    "score": 1170,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'mercury-2'; 955 votes; published 95% interval 1145.5–1194.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "glm-4-plus-0111",
    "bench": "lmarena-text",
    "score": 1342.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'glm-4-plus-0111'; 5,760 votes; published 95% interval 1334.3–1351.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-turbo-0110",
    "bench": "lmarena-text",
    "score": 1340.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-turbo-0110'; 2,290 votes; published 95% interval 1329.2–1352.2. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-2-16k-exp-202412",
    "bench": "lmarena-text",
    "score": 1333.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'step-2-16k-exp-202412'; 4,833 votes; published 95% interval 1325.0–1342.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-1o-turbo-202506",
    "bench": "lmarena-text",
    "score": 1320,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'step-1o-turbo-202506'; 8,845 votes; published 95% interval 1313.1–1326.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-1o-turbo-202506",
    "bench": "lmarena-vision",
    "score": 1155.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'step-1o-turbo-202506'; 1,897 votes; published 95% interval 1141.5–1170.1. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "step-1o-vision-32k-highres",
    "bench": "lmarena-vision",
    "score": 1124.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'step-1o-vision-32k-highres'; 2,833 votes; published 95% interval 1111.6–1136.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "hunyuan-standard-256k",
    "bench": "lmarena-text",
    "score": 1233.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'hunyuan-standard-256k'; 2,728 votes; published 95% interval 1221.5–1244.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-26-02-10",
    "bench": "lmarena-text",
    "score": 1426.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-26-02-10'; 3,519 votes; published 95% interval 1416.9–1436.3. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-26-01-10",
    "bench": "lmarena-text",
    "score": 1413.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-26-01-10'; 3,534 votes; published 95% interval 1403.4–1422.8. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-12-10",
    "bench": "lmarena-text",
    "score": 1393.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-12-10'; 3,728 votes; published 95% interval 1384.5–1403.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-11-10",
    "bench": "lmarena-text",
    "score": 1364.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-11-10'; 25,995 votes; published 95% interval 1360.0–1368.5. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-10-20",
    "bench": "lmarena-text",
    "score": 1348.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-10-20'; 11,590 votes; published 95% interval 1342.1–1354.4. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "amazon-nova-experimental-chat-10-09",
    "bench": "lmarena-text",
    "score": 1347,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'amazon-nova-experimental-chat-10-09'; 2,765 votes; published 95% interval 1336.0–1357.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "ideogram-4-5",
    "bench": "lmarena-image-edit",
    "score": 1347.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'ideogram-4.5'; 19,318 votes; published 95% interval 1341.9–1352.8; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Refresh of the row that cited revision 46919c46 (score 1351.4), same exact entry, so each image run sits on one snapshot (held in pass 8; applied under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "step-5-preview",
    "bench": "lmarena-webdev",
    "score": 1570.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-01",
    "note": "webdev, category overall, split latest; exact entry 'step-5-preview-high'; 2,506 votes; published 95% interval 1557.4–1582.9. Refreshed from pinned October dataset; replaces the older snapshot for this model/task."
  },
  {
    "model": "llama-2-7b",
    "bench": "mmlu",
    "score": 0.444,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "hellaswag",
    "score": 0.7709999999999999,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "winogrande",
    "score": 0.695,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "piqa",
    "score": 0.779,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-easy",
    "score": 0.687,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.43200000000000005,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "natural-questions",
    "score": 0.175,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "triviaqa",
    "score": 0.5660000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "humaneval",
    "score": 0.11599999999999999,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "mbpp",
    "score": 0.261,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "math",
    "score": 0.039,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-7b",
    "bench": "gsm8k",
    "score": 0.16,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "mmlu",
    "score": 0.556,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "hellaswag",
    "score": 0.807,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "winogrande",
    "score": 0.7290000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "piqa",
    "score": 0.8079999999999999,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-easy",
    "score": 0.752,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-challenge",
    "score": 0.488,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "natural-questions",
    "score": 0.16699999999999998,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "triviaqa",
    "score": 0.64,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "humaneval",
    "score": 0.18899999999999997,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "mbpp",
    "score": 0.354,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "math",
    "score": 0.06,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "gsm8k",
    "score": 0.34299999999999997,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "mmlu",
    "score": 0.568,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "hellaswag",
    "score": 0.8370000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "winogrande",
    "score": 0.762,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "piqa",
    "score": 0.8220000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "arc-easy",
    "score": 0.7959999999999999,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "arc-challenge",
    "score": 0.544,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "natural-questions",
    "score": 0.24100000000000002,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "triviaqa",
    "score": 0.685,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "humaneval",
    "score": 0.25,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "mbpp",
    "score": 0.409,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "math",
    "score": 0.084,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-33b",
    "bench": "gsm8k",
    "score": 0.441,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "mmlu",
    "score": 0.6990000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "hellaswag",
    "score": 0.8540000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "winogrande",
    "score": 0.804,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "piqa",
    "score": 0.826,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "arc-easy",
    "score": 0.799,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "arc-challenge",
    "score": 0.565,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "natural-questions",
    "score": 0.254,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "triviaqa",
    "score": 0.73,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "humaneval",
    "score": 0.293,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "mbpp",
    "score": 0.498,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "math",
    "score": 0.138,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "llama-2-70b",
    "bench": "gsm8k",
    "score": 0.696,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "independent",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mmlu",
    "score": 0.625,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "hellaswag",
    "score": 0.81,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "winogrande",
    "score": 0.742,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "piqa",
    "score": 0.8220000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-easy",
    "score": 0.805,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-challenge",
    "score": 0.5489999999999999,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "natural-questions",
    "score": 0.23199999999999998,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "triviaqa",
    "score": 0.625,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "humaneval",
    "score": 0.262,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mbpp",
    "score": 0.502,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "math",
    "score": 0.127,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "gsm8k",
    "score": 0.5,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "mmlu",
    "score": 0.706,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "hellaswag",
    "score": 0.8440000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "winogrande",
    "score": 0.772,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "piqa",
    "score": 0.836,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "arc-easy",
    "score": 0.831,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "arc-challenge",
    "score": 0.597,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "natural-questions",
    "score": 0.306,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "triviaqa",
    "score": 0.715,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "humaneval",
    "score": 0.402,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "mbpp",
    "score": 0.607,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "math",
    "score": 0.284,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "gsm8k",
    "score": 0.7440000000000001,
    "source": "https://arxiv.org/html/2401.04088v1#S3.T2",
    "by": "lab",
    "date": "2024-01-08",
    "note": "Mixtral of Experts, Table 2: all six base models re-evaluated by the authors with the same pipeline. Table 3 uses different shot settings and is excluded. MBPP uses the hand-verified subset; TriviaQA without Wikipedia context.",
    "source_id": "paper-mixtral-2401.04088v1"
  },
  {
    "model": "codellama-7b",
    "bench": "mmlu",
    "score": 0.369,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "hellaswag",
    "score": 0.629,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "winogrande",
    "score": 0.623,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "piqa",
    "score": 0.728,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "arc-easy",
    "score": 0.594,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "arc-challenge",
    "score": 0.345,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "natural-questions",
    "score": 0.11,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "triviaqa",
    "score": 0.349,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "humaneval",
    "score": 0.311,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "mbpp",
    "score": 0.525,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "math",
    "score": 0.052000000000000005,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "codellama-7b",
    "bench": "gsm8k",
    "score": 0.20800000000000002,
    "source": "https://arxiv.org/html/2310.06825v1#S3.T2",
    "source_id": "paper-2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B, Table 2; authors reran all models. Code-Llama 7B is the code-pretrained base checkpoint, not Code Llama Instruct or Python. Original paper benchmark/shot settings retained."
  },
  {
    "model": "gemma-2b-base",
    "bench": "mmlu",
    "score": 0.423,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot, top-1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "mmlu",
    "score": 0.643,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot, top-1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "hellaswag",
    "score": 0.7140000000000001,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "hellaswag",
    "score": 0.812,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "piqa",
    "score": 0.773,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "piqa",
    "score": 0.812,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "social-iqa",
    "score": 0.47,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "social-iqa",
    "score": 0.49700000000000005,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "social-iqa",
    "score": 0.518,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "boolq",
    "score": 0.8320000000000001,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "boolq",
    "score": 0.6940000000000001,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "boolq",
    "score": 0.8320000000000001,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 0-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "winogrande",
    "score": 0.654,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; partial scoring. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "winogrande",
    "score": 0.723,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; partial scoring. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "commonsenseqa",
    "score": 0.6629999999999999,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 7-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "commonsenseqa",
    "score": 0.653,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 7-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "commonsenseqa",
    "score": 0.713,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 7-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "openbookqa",
    "score": 0.478,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "openbookqa",
    "score": 0.528,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "arc-easy",
    "score": 0.732,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "arc-easy",
    "score": 0.815,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "arc-challenge",
    "score": 0.42100000000000004,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "arc-challenge",
    "score": 0.532,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "triviaqa",
    "score": 0.532,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "triviaqa",
    "score": 0.634,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "natural-questions",
    "score": 0.125,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "natural-questions",
    "score": 0.23,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 5-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "humaneval",
    "score": 0.22,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; pass@1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "humaneval",
    "score": 0.32299999999999995,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; pass@1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mbpp",
    "score": 0.402,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 3-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "mbpp",
    "score": 0.292,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 3-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "mbpp",
    "score": 0.444,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 3-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "gsm8k",
    "score": 0.354,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; maj@1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "gsm8k",
    "score": 0.177,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; maj@1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "gsm8k",
    "score": 0.46399999999999997,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; maj@1. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "math",
    "score": 0.11800000000000001,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 4-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "math",
    "score": 0.243,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; 4-shot. Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "agieval-gemma-en",
    "score": 0.41200000000000003,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "agieval-gemma-en",
    "score": 0.242,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "agieval-gemma-en",
    "score": 0.41700000000000004,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "bbh",
    "score": 0.561,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "independent",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-2b-base",
    "bench": "bbh",
    "score": 0.35200000000000004,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gemma-7b-base",
    "bench": "bbh",
    "score": 0.551,
    "source": "https://arxiv.org/html/2403.08295v4#S3.T6",
    "source_id": "paper-gemma-2403.08295v4",
    "by": "lab",
    "date": "2024-04-19",
    "note": "Gemma technical report, Table 6; . Gemma own-model cells and starred Mistral reruns only; all copied Llama and unstarred Mistral values excluded."
  },
  {
    "model": "gpt-4-0613",
    "bench": "mmlu",
    "score": 0.8140000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-4-0613",
    "bench": "gsm8k",
    "score": 0.95,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-4-0613",
    "bench": "bbh",
    "score": 0.8909999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-4-0613",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.652,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-4-0613",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.87,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "mmlu",
    "score": 0.657,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "gsm8k",
    "score": 0.765,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "bbh",
    "score": 0.708,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.512,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.88,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "mmlu",
    "score": 0.679,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "gsm8k",
    "score": 0.76,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "bbh",
    "score": 0.6609999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.519,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.884,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "mmlu",
    "score": 0.586,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "gsm8k",
    "score": 0.28,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "bbh",
    "score": 0.449,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.237,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.5429999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "mmlu",
    "score": 0.65,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "gsm8k",
    "score": 0.655,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "bbh",
    "score": 0.6559999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.382,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.6609999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "mmlu",
    "score": 0.46799999999999997,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "gsm8k",
    "score": 0.12,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "bbh",
    "score": 0.256,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.22699999999999998,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.24,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "mmlu",
    "score": 0.532,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "gsm8k",
    "score": 0.09,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "bbh",
    "score": 0.40299999999999997,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.321,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.331,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "mmlu",
    "score": 0.609,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "gsm8k",
    "score": 0.59,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "bbh",
    "score": 0.49,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.444,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.521,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-7b",
    "bench": "mmlu",
    "score": 0.504,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-7b",
    "bench": "gsm8k",
    "score": 0.34,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-7b",
    "bench": "bbh",
    "score": 0.485,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.46399999999999997,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.369,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "mmlu",
    "score": 0.507,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "gsm8k",
    "score": 0.345,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "bbh",
    "score": 0.455,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.445,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.4,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-13b",
    "bench": "mmlu",
    "score": 0.5539999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-13b",
    "bench": "gsm8k",
    "score": 0.46,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-13b",
    "bench": "bbh",
    "score": 0.495,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-13b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.532,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-13b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.49,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "mmlu",
    "score": 0.5529999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "gsm8k",
    "score": 0.495,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "bbh",
    "score": 0.494,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.397,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.489,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-70b",
    "bench": "mmlu",
    "score": 0.6729999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-70b",
    "bench": "gsm8k",
    "score": 0.73,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-70b",
    "bench": "bbh",
    "score": 0.684,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-70b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.536,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-70b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.685,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "mmlu",
    "score": 0.6779999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "gsm8k",
    "score": 0.715,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "bbh",
    "score": 0.66,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.358,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.6890000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 1, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b",
    "bench": "mmlu",
    "score": 0.418,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b",
    "bench": "gsm8k",
    "score": 0.12,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b",
    "bench": "bbh",
    "score": 0.39299999999999996,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.512,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.268,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b",
    "bench": "mmlu",
    "score": 0.52,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b",
    "bench": "gsm8k",
    "score": 0.25,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b",
    "bench": "bbh",
    "score": 0.489,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.565,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-13b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.325,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b",
    "bench": "mmlu",
    "score": 0.645,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b",
    "bench": "gsm8k",
    "score": 0.555,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b",
    "bench": "bbh",
    "score": 0.66,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.626,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "llama-2-70b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.601,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 5, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b",
    "bench": "mmlu",
    "score": 0.33799999999999997,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b",
    "bench": "gsm8k",
    "score": 0.12,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b",
    "bench": "bbh",
    "score": 0.434,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.47600000000000003,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.5870000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "mmlu",
    "score": 0.415,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "gsm8k",
    "score": 0.17,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "bbh",
    "score": 0.384,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.41600000000000004,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.6409999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-7b",
    "bench": "mmlu",
    "score": 0.43700000000000006,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-7b",
    "bench": "gsm8k",
    "score": 0.33,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-7b",
    "bench": "bbh",
    "score": 0.491,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.526,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.6890000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b",
    "bench": "mmlu",
    "score": 0.375,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b",
    "bench": "gsm8k",
    "score": 0.22,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b",
    "bench": "bbh",
    "score": 0.495,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.521,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.698,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "mmlu",
    "score": 0.433,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "gsm8k",
    "score": 0.23,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "bbh",
    "score": 0.48,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.37799999999999995,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.6920000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-13b",
    "bench": "mmlu",
    "score": 0.45899999999999996,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-13b",
    "bench": "gsm8k",
    "score": 0.41,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-13b",
    "bench": "bbh",
    "score": 0.528,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-13b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.557,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-13b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.762,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b",
    "bench": "mmlu",
    "score": 0.474,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b",
    "bench": "gsm8k",
    "score": 0.35,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b",
    "bench": "bbh",
    "score": 0.57,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.5710000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.7759999999999999,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "mmlu",
    "score": 0.509,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "gsm8k",
    "score": 0.38,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "bbh",
    "score": 0.5920000000000001,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.551,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.765,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "independent",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-34b",
    "bench": "mmlu",
    "score": 0.536,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-34b",
    "bench": "gsm8k",
    "score": 0.54,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-34b",
    "bench": "bbh",
    "score": 0.643,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-34b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.606,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "code-tulu-2-34b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.825,
    "source": "https://arxiv.org/html/2311.10702v2",
    "source_id": "paper-tulu2-2311.10702v2",
    "by": "lab",
    "date": "2023-11-20",
    "note": "Tulu 2 paper, Table 6, authors evaluation suite. MMLU 0-shot; GSM8K 8-shot CoT; BBH 3-shot CoT; TyDiQA gold passage 1-shot F1; CodexEval pass@10 (not pass@1). Same cells repeated in ablation tables imported once."
  },
  {
    "model": "claude-opus-5-5",
    "bench": "lmarena-text",
    "score": 1503.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'claude-opus-5.5-high'; 4,552 votes; published 95% interval 1494.6–1512.8. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "lmarena-text",
    "score": 1480.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'mimo-v2.6-pro'; 4,056 votes; published 95% interval 1470.7–1489.5. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "lmarena-text",
    "score": 1474.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'deepseek-v4.1-flash-max'; 8,728 votes; published 95% interval 1467.5–1481.4. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "gpt-6-sol",
    "bench": "lmarena-text",
    "score": 1457.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'gpt-6-sol-max'; 8,787 votes; published 95% interval 1450.2–1464.2. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "gpt-6-luna",
    "bench": "lmarena-text",
    "score": 1442.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'gpt-6-luna-max'; 10,014 votes; published 95% interval 1436.3–1449.4. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "grok-4-7",
    "bench": "lmarena-text",
    "score": 1442.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall; exact entry 'grok-4.7-xhigh'; 6,405 votes; published 95% interval 1434.7–1450.2. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "lmarena-vision",
    "score": 1296,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall; exact entry 'gemini-3.7-flash-high'; 3,814 votes; published 95% interval 1285.5–1306.5. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "lmarena-vision",
    "score": 1290.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall; exact entry 'gemini-3.8-flash-high'; 3,086 votes; published 95% interval 1278.8–1301.9. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "lmarena-vision",
    "score": 1245.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall; exact entry 'mimo-v2.6-pro'; 1,787 votes; published 95% interval 1230.6–1259.9. Newly available task measurement; highest reviewed reasoning setting in this task. No lower-setting duplication."
  },
  {
    "model": "claude-fable-5-1",
    "bench": "arena-agent-ips-202610",
    "score": 0.14310371476177605,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Fable 5.1 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.12415154304102166, 0.16205588648253044]; 15,130 sessions, 1,683,381 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-opus-5-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.13817012143011934,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Opus 5.5 (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.11645483256257871, 0.15988541029765996]; 5,359 sessions, 754,031 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.12520807893993158,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Sonnet 5.5 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.09428887464329698, 0.15612728323656616]; 5,219 sessions, 483,211 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-6-astra",
    "bench": "arena-agent-ips-202610",
    "score": 0.12271283712595349,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 6 Astra (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.100413035461178, 0.14501263879072898]; 11,717 sessions, 888,641 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "arena-agent-ips-202610",
    "score": 0.11228181892445037,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 6.1 Sol (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.08464406868737293, 0.1399195691615278]; 6,278 sessions, 379,372 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-6-sol",
    "bench": "arena-agent-ips-202610",
    "score": 0.09711069971200803,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 6 Sol (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.07344719892801793, 0.12077420049599813]; 6,106 sessions, 994,367 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-fable-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.08212369773489381,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Fable 5 (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.06995616515766467, 0.09429123031212294]; 41,832 sessions, 2,603,067 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-opus-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.07923513870898408,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Opus 5 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.06448680051842494, 0.09398347689954321]; 22,781 sessions, 3,006,257 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gemini-4-argon",
    "bench": "arena-agent-ips-202610",
    "score": 0.07566193515418262,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Gemini 4 Argon (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.056347909468084184, 0.09497596084028107]; 10,030 sessions, 587,353 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-opus-4-8",
    "bench": "arena-agent-ips-202610",
    "score": 0.06641949587363769,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Opus 4.8 (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.05299622583850096, 0.07984276590877441]; 41,132 sessions, 2,431,185 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "arena-agent-ips-202610",
    "score": 0.06450039456435545,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 5.6 Sol (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.051676213416725576, 0.07732457571198534]; 35,896 sessions, 4,348,715 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "claude-sonnet-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.043985032829503515,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Claude Sonnet 5 (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.028172889685818808, 0.05979717597318822]; 32,467 sessions, 3,612,898 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "kimi-k3",
    "bench": "arena-agent-ips-202610",
    "score": 0.04184006011674849,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Kimi K3 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.03647210983229049, 0.04720801040120649]; 132,505 sessions, 12,305,667 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-5-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.04154253554559495,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 5.5 (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.03129060767491741, 0.05179446341627249]; 55,006 sessions, 3,060,353 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "grok-4-7",
    "bench": "arena-agent-ips-202610",
    "score": 0.0403638153342592,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Grok 4.7 (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.024323979592707053, 0.05640365107581135]; 11,591 sessions, 1,656,641 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "arena-agent-ips-202610",
    "score": 0.040249812172174916,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Deepseek V4.1 Flash (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.0355845372330827, 0.044915087111267134]; 112,938 sessions, 16,932,092 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "muse-spark-1-3",
    "bench": "arena-agent-ips-202610",
    "score": 0.039958616826527994,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Muse Spark 1.3 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.033982590688229, 0.04593464296482699]; 64,671 sessions, 6,139,426 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "hy4-preview",
    "bench": "arena-agent-ips-202610",
    "score": 0.03953414370418734,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Hy4 preview'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.03160432913148551, 0.04746395827688917]; 59,388 sessions, 5,108,255 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "glm-5-2",
    "bench": "arena-agent-ips-202610",
    "score": 0.034795448332433254,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GLM 5.2 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.027614565013024867, 0.04197633165184164]; 86,172 sessions, 6,684,620 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "arena-agent-ips-202610",
    "score": 0.02965042916523071,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Gemini 3.8 Flash (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.021228913793662735, 0.038071944536798685]; 34,396 sessions, 4,279,699 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "arena-agent-ips-202610",
    "score": 0.025115313846606382,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Qwen3.8 Max'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.01887951677229076, 0.03135111092092201]; 52,824 sessions, 4,577,924 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "glm-5-3",
    "bench": "arena-agent-ips-202610",
    "score": 0.02358234028156805,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GLM 5.3 (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.016740103879208115, 0.030424576683927985]; 86,384 sessions, 9,155,315 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-6-luna",
    "bench": "arena-agent-ips-202610",
    "score": 0.013523401995501593,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 6 Luna (Max)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.002807534834065562, 0.024239269156937623]; 29,241 sessions, 3,443,780 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "grok-4-6",
    "bench": "arena-agent-ips-202610",
    "score": 0.012802635481312387,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Grok 4.6 (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.002784269876943515, 0.02282100108568126]; 27,380 sessions, 3,131,725 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "grok-4-5",
    "bench": "arena-agent-ips-202610",
    "score": 0.01208390746446243,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Grok 4.5'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.0019617931260805486, 0.02220602180284431]; 41,920 sessions, 2,418,629 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "arena-agent-ips-202610",
    "score": 0.0118540717701348,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'DeepSeek V4 Pro (High) (0813)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.0047129205344373995, 0.028421064074707]; 7,090 sessions, 797,284 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-5-4",
    "bench": "arena-agent-ips-202610",
    "score": 0.01053799653803703,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 5.4 (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [0.0013255801637096012, 0.01975041291236446]; 81,597 sessions, 4,332,904 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "step-5-preview",
    "bench": "arena-agent-ips-202610",
    "score": 0.004831647071142874,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Step 5 Preview'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.009863944134801792, 0.019527238277087543]; 9,453 sessions, 1,125,598 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "arena-agent-ips-202610",
    "score": 0.0040286217871811645,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 5.6 Terra (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.00755816966171599, 0.015615413236078318]; 23,457 sessions, 1,686,712 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "glm-5-3-flash",
    "bench": "arena-agent-ips-202610",
    "score": -0.004366471879715806,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GLM 5.3 Flash'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.009040788597539741, 0.00030784483810813047]; 80,836 sessions, 9,533,773 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "arena-agent-ips-202610",
    "score": -0.0072646022565682755,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Qwen3.8 Flash Next'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.013552000993865224, -0.0009772035192713267]; 83,198 sessions, 11,249,221 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "arena-agent-ips-202610",
    "score": -0.01165468069044633,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'GPT 5.6 Luna (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.018391575882733734, -0.0049177854981589265]; 42,599 sessions, 4,028,029 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "arena-agent-ips-202610",
    "score": -0.014851314462290235,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Gemini 3.7 Flash (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.022204972726663337, -0.007497656197917133]; 54,594 sessions, 5,048,920 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "qwen3-8-27b",
    "bench": "arena-agent-ips-202610",
    "score": -0.016562315929168292,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Qwen 3.8 27B'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.022521536082842943, -0.010603095775493641]; 57,343 sessions, 6,972,838 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "muse-spark-1-2",
    "bench": "arena-agent-ips-202610",
    "score": -0.03272818511060355,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Muse Spark 1.2 (xHigh)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.03946558677148422, -0.025990783449722877]; 48,150 sessions, 2,884,900 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "muse-spark-1-1",
    "bench": "arena-agent-ips-202610",
    "score": -0.048840592019033596,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Muse Spark 1.1'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.053302077309958236, -0.04437910672810896]; 142,711 sessions, 7,612,456 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "qwen3-7-max",
    "bench": "arena-agent-ips-202610",
    "score": -0.05129166602323023,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Qwen3.7 Max'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.060609075119189545, -0.041974256927270914]; 40,438 sessions, 2,207,521 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "hy3",
    "bench": "arena-agent-ips-202610",
    "score": -0.05379482794866417,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Hy3'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.06591497360554492, -0.041674682291783414]; 29,677 sessions, 1,171,568 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "minimax-m3",
    "bench": "arena-agent-ips-202610",
    "score": -0.06874439858372874,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Minimax M3'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.0775638626097857, -0.05992493455767178]; 41,267 sessions, 3,338,791 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "qwen3-7-plus",
    "bench": "arena-agent-ips-202610",
    "score": -0.07154412770519575,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Qwen3.7 Plus'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.08610369957455825, -0.056984555835833235]; 21,722 sessions, 1,144,063 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "arena-agent-ips-202610",
    "score": -0.0752116255861309,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Mimo V2.5 Pro'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.08410485818840951, -0.06631839298385228]; 45,239 sessions, 2,309,389 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "arena-agent-ips-202610",
    "score": -0.07683234208284252,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Gemini 3.6 Flash (High)'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.08741234081641375, -0.0662523433492713]; 22,913 sessions, 1,523,777 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "arena-agent-ips-202610",
    "score": -0.07711672238819436,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Gemini 3.1 Pro Preview'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.08690533911564136, -0.06732810566074737]; 86,483 sessions, 3,446,977 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "inkling-small",
    "bench": "arena-agent-ips-202610",
    "score": -0.1032662436055972,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Inkling Small'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.11732701403245874, -0.08920547317873567]; 13,273 sessions, 503,653 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "inkling",
    "bench": "arena-agent-ips-202610",
    "score": -0.10859866066096427,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Inkling'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.11902091761974418, -0.09817640370218436]; 42,813 sessions, 1,798,990 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "arena-agent-ips-202610",
    "score": -0.12382085550188522,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Mistral Medium 3.5'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.1390025386941619, -0.10863917230960854]; 11,156 sessions, 437,621 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "solar-pro-4",
    "bench": "arena-agent-ips-202610",
    "score": -0.15902618594687917,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet",
    "source_id": "lmarena-dataset",
    "by": "independent",
    "date": "2026-10-02",
    "note": "agent latest overall; exact entry 'Solar Pro 4'; highest available reasoning effort for this model in this task. Aggregate IPS net improvement, not Elo; 95% CI [-0.17753581013274417, -0.14051656176101418]; 9,506 sessions, 427,705 observations. October 1, 2026 signal definitions. Lower effort and the five constituent signals are not duplicated."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "arc-challenge",
    "score": 0.257679180887372,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "arc-easy",
    "score": 0.42045454545454547,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "boolq",
    "score": 0.5764525993883792,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "hellaswag",
    "score": 0.38309101772555265,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "lambada-openai",
    "score": 0.4172326799922375,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "openbookqa",
    "score": 0.17,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "piqa",
    "score": 0.6381936887921654,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "sciq",
    "score": 0.717,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "social-iqa",
    "score": 0.3561924257932446,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "truthfulqa-mc2",
    "score": 0.4052844601694033,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "winogrande",
    "score": 0.526440410418311,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "arc-challenge",
    "score": 0.27047781569965873,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "arc-easy",
    "score": 0.44865319865319864,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "boolq",
    "score": 0.600611620795107,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "hellaswag",
    "score": 0.41216889065923124,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "lambada-openai",
    "score": 0.5511352610130021,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "openbookqa",
    "score": 0.214,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "piqa",
    "score": 0.6675734494015234,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "sciq",
    "score": 0.801,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "social-iqa",
    "score": 0.394575230296827,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.3995908363542637,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "winogrande",
    "score": 0.5011838989739542,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "arc-challenge",
    "score": 0.40017064846416384,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "arc-easy",
    "score": 0.6771885521885522,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "boolq",
    "score": 0.7562691131498471,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "hellaswag",
    "score": 0.7389962158932484,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "lambada-openai",
    "score": 0.7063846303124394,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "openbookqa",
    "score": 0.314,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "piqa",
    "score": 0.7921653971708379,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "sciq",
    "score": 0.948,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "social-iqa",
    "score": 0.41914022517911975,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "winogrande",
    "score": 0.665351223362273,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-3b-4e1t.json",
    "source_id": "stablelm-original-eval-json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=stablelm-3b-4e1t,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "arc-challenge",
    "score": 0.3651877133105802,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "arc-easy",
    "score": 0.6224747474747475,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "boolq",
    "score": 0.654434250764526,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "hellaswag",
    "score": 0.6624178450507867,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "lambada-openai",
    "score": 0.6823209780710265,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "openbookqa",
    "score": 0.29,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "piqa",
    "score": 0.7557127312295974,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "sciq",
    "score": 0.915,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "social-iqa",
    "score": 0.4109518935516888,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-j-6b",
    "bench": "winogrande",
    "score": 0.6416732438831886,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-j-6B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=EleutherAI/gpt-j-6B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "arc-challenge",
    "score": 0.40784982935153585,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "arc-easy",
    "score": 0.6868686868686869,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "boolq",
    "score": 0.6948012232415902,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "hellaswag",
    "score": 0.714299940250946,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "lambada-openai",
    "score": 0.7197748884145159,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "openbookqa",
    "score": 0.298,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "piqa",
    "score": 0.7742110990206746,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "sciq",
    "score": 0.931,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "social-iqa",
    "score": 0.4196519959058342,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-neox-20b",
    "bench": "winogrande",
    "score": 0.6614048934490924,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_gpt-neox-20B.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=EleutherAI/gpt-neox-20B,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "arc-challenge",
    "score": 0.3506825938566553,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "arc-easy",
    "score": 0.6372053872053872,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "boolq",
    "score": 0.6730886850152905,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "hellaswag",
    "score": 0.673770165305716,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "lambada-openai",
    "score": 0.7063846303124394,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "openbookqa",
    "score": 0.264,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "piqa",
    "score": 0.7627856365614799,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "sciq",
    "score": 0.902,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "social-iqa",
    "score": 0.4201637666325486,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-12b",
    "bench": "winogrande",
    "score": 0.6400947119179163,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-12b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=EleutherAI/pythia-12b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-challenge",
    "score": 0.3532423208191126,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "arc-easy",
    "score": 0.6106902356902357,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "boolq",
    "score": 0.6400611620795107,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "hellaswag",
    "score": 0.6388169687313284,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "lambada-openai",
    "score": 0.670095090238696,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "openbookqa",
    "score": 0.258,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "piqa",
    "score": 0.750816104461371,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "sciq",
    "score": 0.898,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "social-iqa",
    "score": 0.40736949846468784,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "pythia-6-9b",
    "bench": "winogrande",
    "score": 0.606156274664562,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI_pythia-6.9b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=EleutherAI/pythia-6.9b,dtype=float16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "arc-challenge",
    "score": 0.3037542662116041,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "arc-easy",
    "score": 0.5328282828282829,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "boolq",
    "score": 0.617125382262997,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "hellaswag",
    "score": 0.545309699263095,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "lambada-openai",
    "score": 0.5173685231903745,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "openbookqa",
    "score": 0.218,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "piqa",
    "score": 0.705658324265506,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "sciq",
    "score": 0.891,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "social-iqa",
    "score": 0.4017400204708291,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-3b",
    "bench": "winogrande",
    "score": 0.584846093133386,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-3b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-3b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "arc-challenge",
    "score": 0.33532423208191126,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "arc-easy",
    "score": 0.5736531986531986,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "boolq",
    "score": 0.6284403669724771,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "hellaswag",
    "score": 0.6228838876717785,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "lambada-openai",
    "score": 0.5755870366776635,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "openbookqa",
    "score": 0.252,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "piqa",
    "score": 0.7274211099020674,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "sciq",
    "score": 0.901,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "social-iqa",
    "score": 0.4211873080859775,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "bloom-7b1",
    "bench": "winogrande",
    "score": 0.6464088397790055,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/bigscience-bloom-7b1.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=bigscience/bloom-7b1,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "arc-challenge",
    "score": 0.31313993174061433,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "arc-easy",
    "score": 0.5429292929292929,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "boolq",
    "score": 0.6033639143730887,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "hellaswag",
    "score": 0.6059549890460068,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "lambada-openai",
    "score": 0.6357461672811954,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "openbookqa",
    "score": 0.25,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "piqa",
    "score": 0.7383025027203483,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "sciq",
    "score": 0.858,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "social-iqa",
    "score": 0.4083930399181167,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-2-7b",
    "bench": "winogrande",
    "score": 0.6101026045777427,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-2.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-2.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "arc-challenge",
    "score": 0.34812286689419797,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "arc-easy",
    "score": 0.6014309764309764,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "boolq",
    "score": 0.6602446483180429,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "hellaswag",
    "score": 0.6719776936865166,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "lambada-openai",
    "score": 0.6764991267222977,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "openbookqa",
    "score": 0.276,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "piqa",
    "score": 0.7633297062023939,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "sciq",
    "score": 0.901,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "social-iqa",
    "score": 0.4263050153531218,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "opt-6-7b",
    "bench": "winogrande",
    "score": 0.6535122336227308,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/facebook-opt-6.7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=facebook/opt-6.7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-challenge",
    "score": 0.492320819112628,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-easy",
    "score": 0.7760942760942761,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "boolq",
    "score": 0.8051987767584098,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "hellaswag",
    "score": 0.7935670185222067,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "lambada-openai",
    "score": 0.767708131185717,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "openbookqa",
    "score": 0.354,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "piqa",
    "score": 0.7905331882480957,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "sciq",
    "score": 0.945,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "social-iqa",
    "score": 0.4278403275332651,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-13b",
    "bench": "winogrande",
    "score": 0.7221783741120757,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-13b-hf.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-13b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.4616040955631399,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-easy",
    "score": 0.7453703703703703,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "boolq",
    "score": 0.7773700305810397,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "hellaswag",
    "score": 0.7594104760007967,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "lambada-openai",
    "score": 0.7347176402095866,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "openbookqa",
    "score": 0.314,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "piqa",
    "score": 0.7774755168661589,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "sciq",
    "score": 0.936,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "social-iqa",
    "score": 0.43500511770726713,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.38967559882659686,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "llama-2-7b",
    "bench": "winogrande",
    "score": 0.6961325966850829,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/meta-llama-Llama-2-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=meta-llama/Llama-2-7b-hf,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "arc-challenge",
    "score": 0.4803754266211604,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "arc-easy",
    "score": 0.7314814814814815,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "boolq",
    "score": 0.7452599388379205,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "hellaswag",
    "score": 0.6261700856403107,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "lambada-openai",
    "score": 0.527459732194838,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "openbookqa",
    "score": 0.376,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "piqa",
    "score": 0.7633297062023939,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "sciq",
    "score": 0.932,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "social-iqa",
    "score": 0.5537359263050153,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "phi-1-5",
    "bench": "winogrande",
    "score": 0.7253354380426204,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/microsoft-phi-1_5.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=microsoft/phi-1_5,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "arc-challenge",
    "score": 0.43686006825938567,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "arc-easy",
    "score": 0.7079124579124579,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "boolq",
    "score": 0.735474006116208,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "hellaswag",
    "score": 0.7634933280223063,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "lambada-openai",
    "score": 0.7455850960605472,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "openbookqa",
    "score": 0.306,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "piqa",
    "score": 0.794885745375408,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "sciq",
    "score": 0.94,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "social-iqa",
    "score": 0.42067553735926305,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "falcon-7b",
    "bench": "winogrande",
    "score": 0.6724546172059984,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/tiiuae_falcon-7b.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=tiiuae/falcon-7b,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "arc-challenge",
    "score": 0.39419795221843,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "arc-easy",
    "score": 0.6919191919191919,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "boolq",
    "score": 0.7076452599388379,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "hellaswag",
    "score": 0.7033459470225055,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "lambada-openai",
    "score": 0.713370851930914,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "openbookqa",
    "score": 0.29,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "piqa",
    "score": 0.7714907508161044,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "sciq",
    "score": 0.927,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "social-iqa",
    "score": 0.4257932446264074,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "truthfulqa-mc2",
    "score": 0.3301333660822527,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "winogrande",
    "score": 0.6432517758484609,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/togethercomputer-RedPajama-INCITE-7B-Base2.json",
    "source_id": "stablelm-original-eval-json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=togethercomputer/RedPajama-INCITE-7B-Base,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot)."
  },
  {
    "model": "gpt-2-small",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.079,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 7.9%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-small",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.515,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 51.5%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-small",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.311,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 31.1%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-medium",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.105,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 10.5%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-medium",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.47200000000000003,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 47.2%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-medium",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.32,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 32.0%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-large",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.113,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 11.3%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-large",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.47200000000000003,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 47.2%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-large",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.33799999999999997,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 33.8%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-xl",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.11699999999999999,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 11.7%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-xl",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.47100000000000003,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 47.1%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-2-xl",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.37,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 37.0%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-neo-125m",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.079,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 7.9%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-neo-125m",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.47600000000000003,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 47.6%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-neo-125m",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.28800000000000003,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 28.8%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-small",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.065,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 6.5%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-small",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.532,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 53.2%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-small",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.401,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 40.1%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-base",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.10300000000000001,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 10.3%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-base",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.495,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 49.5%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-base",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.423,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 42.3%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-large",
    "bench": "copen-csj-zero-shot-full-prompt",
    "score": 0.14,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CSJ All column: 14.0%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-large",
    "bench": "copen-cpj-zero-shot-full-prompt",
    "score": 0.498,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CPJ All column: 49.8%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "t5-large",
    "bench": "copen-cic-zero-shot-full-prompt",
    "score": 0.426,
    "source": "https://aclanthology.org/2022.emnlp-main.335.pdf",
    "source_id": "copen-emnlp2022-zero-shot",
    "by": "independent",
    "date": "2022-12-07",
    "note": "Table 11 (printed p. 5028), CIC All column: 42.6%. Zero-shot likelihood of the full prompt; no linear probe or fine-tuning. Table 7 maps the exact original Hugging Face checkpoint. Same preselected All protocol across every model; not the family-specific best span from the main table."
  },
  {
    "model": "gpt-oss-120b",
    "bench": "tau2-retail",
    "score": 0.614,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column gpt-oss-120b (reasoning setting not specified in column). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "61.4",
      "column_index": 4,
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "tau2-retail",
    "score": 0.5993999999999999,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column deepseek-r1-0528 (reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "59.94",
      "column_index": 5,
      "scale_divisor": 100
    }
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "tau2-retail",
    "score": 0.6169,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column gemini-2-5-flash-preview-09-2025 (September 2025 checkpoint; thinking budget not specified). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "61.69",
      "column_index": 6,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-retail",
    "score": 0.7339,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column gpt-5-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "73.39",
      "column_index": 7,
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-retail",
    "score": 0.698,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column claude-sonnet-4-5 (thinking). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "69.8",
      "column_index": 8,
      "scale_divisor": 100
    }
  },
  {
    "model": "o3-mini",
    "bench": "tau2-retail",
    "score": 0.7573000000000001,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Retail row, column o3-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Retail",
      "raw_value": "75.73",
      "column_index": 9,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "tau2-airline",
    "score": 0.45299999999999996,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column gpt-oss-120b (reasoning setting not specified in column). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "45.3",
      "column_index": 4,
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "tau2-airline",
    "score": 0.4733,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column deepseek-r1-0528 (reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "47.33",
      "column_index": 5,
      "scale_divisor": 100
    }
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "tau2-airline",
    "score": 0.5666,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column gemini-2-5-flash-preview-09-2025 (September 2025 checkpoint; thinking budget not specified). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "56.66",
      "column_index": 6,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-airline",
    "score": 0.5932999999999999,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column gpt-5-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "59.33",
      "column_index": 7,
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column claude-sonnet-4-5 (thinking). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "58",
      "column_index": 8,
      "scale_divisor": 100
    }
  },
  {
    "model": "o3-mini",
    "bench": "tau2-airline",
    "score": 0.6133,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Tau2 bench Airline row, column o3-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Tau2 bench Airline",
      "raw_value": "61.33",
      "column_index": 9,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "multi-if-unspecified",
    "score": 0.8295,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column gpt-oss-120b (reasoning setting not specified in column). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "82.95",
      "column_index": 4,
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "multi-if-unspecified",
    "score": 0.7376,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column deepseek-r1-0528 (reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "73.76",
      "column_index": 5,
      "scale_divisor": 100
    }
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "multi-if-unspecified",
    "score": 0.8249,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column gemini-2-5-flash-preview-09-2025 (September 2025 checkpoint; thinking budget not specified). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "82.49",
      "column_index": 6,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "multi-if-unspecified",
    "score": 0.8537,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column gpt-5-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "85.37",
      "column_index": 7,
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "multi-if-unspecified",
    "score": 0.8432,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column claude-sonnet-4-5 (thinking). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "84.32",
      "column_index": 8,
      "scale_divisor": 100
    }
  },
  {
    "model": "o3-mini",
    "bench": "multi-if-unspecified",
    "score": 0.8728,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Multi IF row, column o3-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Multi IF",
      "raw_value": "87.28",
      "column_index": 9,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "swe-bench-verified",
    "score": 0.31,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column gpt-oss-120b (reasoning setting not specified in column). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "31",
      "column_index": 4,
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "swe-bench-verified",
    "score": 0.29600000000000004,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column deepseek-r1-0528 (reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "29.60",
      "column_index": 5,
      "scale_divisor": 100
    }
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "swe-bench-verified",
    "score": 0.342,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column gemini-2-5-flash-preview-09-2025 (September 2025 checkpoint; thinking budget not specified). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "34.20",
      "column_index": 6,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "swe-bench-verified",
    "score": 0.61,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column gpt-5-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "61",
      "column_index": 7,
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "swe-bench-verified",
    "score": 0.642,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column claude-sonnet-4-5 (thinking). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "64.2",
      "column_index": 8,
      "scale_divisor": 100
    }
  },
  {
    "model": "o3-mini",
    "bench": "swe-bench-verified",
    "score": 0.226,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, SWE Bench Verified row, column o3-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "SWE Bench Verified",
      "raw_value": "22.60",
      "column_index": 9,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aider-polyglot",
    "score": 0.42,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column gpt-oss-120b (reasoning setting not specified in column). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "42",
      "column_index": 4,
      "scale_divisor": 100
    }
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aider-polyglot",
    "score": 0.7140000000000001,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column deepseek-r1-0528 (reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "71.40",
      "column_index": 5,
      "scale_divisor": 100
    }
  },
  {
    "model": "gemini-2-5-flash-preview-09-2025",
    "bench": "aider-polyglot",
    "score": 0.4,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column gemini-2-5-flash-preview-09-2025 (September 2025 checkpoint; thinking budget not specified). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "40",
      "column_index": 6,
      "scale_divisor": 100
    }
  },
  {
    "model": "gpt-5-mini",
    "bench": "aider-polyglot",
    "score": 0.716,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column gpt-5-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "71.60",
      "column_index": 7,
      "scale_divisor": 100
    }
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "aider-polyglot",
    "score": 0.78,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column claude-sonnet-4-5 (thinking). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "78",
      "column_index": 8,
      "scale_divisor": 100
    }
  },
  {
    "model": "o3-mini",
    "bench": "aider-polyglot",
    "score": 0.604,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "source_id": "card-final-apriel-1-6-15b-thinker",
    "by": "independent",
    "date": "2026-10-06",
    "note": "ServiceNow internal comparison run, Aider Polyglot row, column o3-mini (high reasoning). Card states benchmarks outside AA Index v3.0 were evaluated internally. Same table/protocol as Apriel 1.6; scaffold versions, token limits and run dates are unspecified. No AA-index measurements or projected scores imported. Date is review date.",
    "mode": "thinking",
    "evidence": {
      "kind": "table",
      "table_index": 0,
      "row_label": "Aider Polyglot",
      "raw_value": "60.40",
      "column_index": 9,
      "scale_divisor": 100
    }
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "mmlu-pro",
    "score": 0.7196,
    "source": "https://github.com/SqueezeBits/FlatQuant-EXAONE4.5/blob/5f1ea65605d1f9301cd7d243b1e8a2c6590d2b46/README.md",
    "source_id": "squeezebits-exaone45-bf16",
    "by": "independent",
    "date": "2026-07-20",
    "note": "Original BF16 LGAI-EXAONE/EXAONE-4.5-33B run using vLLM 0.24.0. 5-shot lm-eval, batch 8, max_length 8192; Overall BF16 = 71.96%. Evaluation run date unspecified; repository snapshot date. This short-output configuration is preserved and not equated with unrestricted reasoning."
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "mmmu-pro-standard-lmms-exaone45",
    "score": 0.1757,
    "source": "https://github.com/SqueezeBits/FlatQuant-EXAONE4.5/blob/5f1ea65605d1f9301cd7d243b1e8a2c6590d2b46/README.md",
    "source_id": "squeezebits-exaone45-bf16",
    "by": "independent",
    "date": "2026-07-20",
    "note": "Original BF16 LGAI-EXAONE/EXAONE-4.5-33B run using vLLM 0.24.0. lmms-eval standard = 17.57%; batch 8, max_new_tokens 512, max_model_len 8192. Evaluation run date unspecified; repository snapshot date. This short-output configuration is preserved and not equated with unrestricted reasoning."
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "mmmu-pro-vision-lmms-exaone45",
    "score": 0.215,
    "source": "https://github.com/SqueezeBits/FlatQuant-EXAONE4.5/blob/5f1ea65605d1f9301cd7d243b1e8a2c6590d2b46/README.md",
    "source_id": "squeezebits-exaone45-bf16",
    "by": "independent",
    "date": "2026-07-20",
    "note": "Original BF16 LGAI-EXAONE/EXAONE-4.5-33B run using vLLM 0.24.0. lmms-eval vision = 21.50%; batch 8, max_new_tokens 512, max_model_len 8192. Evaluation run date unspecified; repository snapshot date. This short-output configuration is preserved and not equated with unrestricted reasoning."
  },
  {
    "model": "dia2-2b",
    "bench": "dailydialog-tts-whisper-large-wer",
    "score": 0.0648,
    "source": "https://arxiv.org/html/2601.20510v1",
    "source_id": "speech-tts-deepfake-age-ccby",
    "by": "independent",
    "date": "2026-01-28",
    "note": "Singh et al., Table 2: Dia2 WER 0.0648, already a fraction. Section 3.1 identifies the Nari Labs 2B streaming system; 4,000 DailyDialog samples per model, Whisper-large + jiwer. Randomized multi-speaker configuration; not Dia1.6B, not Dia2 1B. No SIM inferred."
  },
  {
    "model": "glow-tts-hifigan-gradtts-study",
    "bench": "gradtts2021-ljspeech-mos",
    "score": 0.7775,
    "source": "https://proceedings.mlr.press/v139/popov21a/popov21a.pdf",
    "source_id": "speech-gradtts-icml2021",
    "by": "independent",
    "date": "2021-07-18",
    "note": "Original ICML 2021 Table 2 MOS 4.11 ± 0.07 (95% CI), 40 held-out sentences × 10 AMT Master raters. Same HiFi-GAN vocoder as other Table 2 models (section 4). Stored (MOS−1)/4. Study-specific identity only; no equivalence with another released implementation asserted."
  },
  {
    "model": "tacotron2-hifigan-gradtts-study",
    "bench": "gradtts2021-ljspeech-mos",
    "score": 0.83,
    "source": "https://proceedings.mlr.press/v139/popov21a/popov21a.pdf",
    "source_id": "speech-gradtts-icml2021",
    "by": "independent",
    "date": "2021-07-18",
    "note": "Original ICML 2021 Table 2 MOS 4.32 ± 0.07 (95% CI), 40 held-out sentences × 10 AMT Master raters. Same HiFi-GAN vocoder as other Table 2 models (section 4). Stored (MOS−1)/4. Study-specific identity only; no equivalence with another released implementation asserted."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ar-cer",
    "score": 0.016200000000000003,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Arabic (ar). Raw cer 1.62 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ar-speaker-similarity",
    "score": 0.806,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Arabic (ar). Raw speaker-similarity 0.806. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-zh-cer",
    "score": 0.0317,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Chinese (zh). Raw cer 3.17 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-zh-speaker-similarity",
    "score": 0.833,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Chinese (zh). Raw speaker-similarity 0.833. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-en-cer",
    "score": 0.0037,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; English (en). Raw cer 0.37 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-en-speaker-similarity",
    "score": 0.822,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; English (en). Raw speaker-similarity 0.822. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-en-accented-cer",
    "score": 0.0034000000000000002,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; English (en) accented. Raw cer 0.34 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-en-accented-speaker-similarity",
    "score": 0.746,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; English (en) accented. Raw speaker-similarity 0.746. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-fr-cer",
    "score": 0.0154,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; French (fr). Raw cer 1.54 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-fr-speaker-similarity",
    "score": 0.747,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; French (fr). Raw speaker-similarity 0.747. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-de-cer",
    "score": 0.008,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; German (de). Raw cer 0.80 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-de-speaker-similarity",
    "score": 0.742,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; German (de). Raw speaker-similarity 0.742. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-hi-cer",
    "score": 0.0123,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Hindi (hi). Raw cer 1.23 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-hi-speaker-similarity",
    "score": 0.788,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Hindi (hi). Raw speaker-similarity 0.788. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-it-cer",
    "score": 0.0219,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Italian (it). Raw cer 2.19 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-it-speaker-similarity",
    "score": 0.773,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Italian (it). Raw speaker-similarity 0.773. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ja-cer",
    "score": 0.013999999999999999,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Japanese (ja). Raw cer 1.40 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ja-speaker-similarity",
    "score": 0.775,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Japanese (ja). Raw speaker-similarity 0.775. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ko-cer",
    "score": 0.0269,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Korean (ko). Raw cer 2.69 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-ko-speaker-similarity",
    "score": 0.807,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Korean (ko). Raw speaker-similarity 0.807. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-pt-cer",
    "score": 0.0291,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Portuguese (pt). Raw cer 2.91 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-pt-speaker-similarity",
    "score": 0.753,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Portuguese (pt). Raw speaker-similarity 0.753. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-es-cer",
    "score": 0.006,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Spanish (es). Raw cer 0.60 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-es-speaker-similarity",
    "score": 0.793,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Spanish (es). Raw speaker-similarity 0.793. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-vi-cer",
    "score": 0.0059,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Vietnamese (vi). Raw cer 0.59 percent. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "magpie-heldout-vi-speaker-similarity",
    "score": 0.725,
    "source": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md",
    "source_id": "magpie-v2607-card-evaluations",
    "by": "lab",
    "date": "2026-10-06",
    "note": "NVIDIA original held-out per-language evaluation, v2607 column only; Vietnamese (vi). Raw speaker-similarity 0.725. English uses LibriTTS test-clean; accented English uses GLOBE; remaining languages use held-out publisher sets described in the card. No cross-language overall synthesized and no extra independent studies inferred. Date is review date; run date unspecified."
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "lambada-openai",
    "score": 0.6223,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "hellaswag",
    "score": 0.5579,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "winogrande",
    "score": 0.5769,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "piqa",
    "score": 0.7219,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "pythia-2-8b",
    "bench": "lambada-openai",
    "score": 0.6466,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "pythia-2-8b",
    "bench": "hellaswag",
    "score": 0.5933,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "pythia-2-8b",
    "bench": "winogrande",
    "score": 0.6006,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "pythia-2-8b",
    "bench": "piqa",
    "score": 0.7399,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "independent",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "redpajama-incite-base-3b-v1",
    "bench": "lambada-openai",
    "score": 0.6541,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "lab",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "redpajama-incite-base-3b-v1",
    "bench": "hellaswag",
    "score": 0.6317,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "lab",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "redpajama-incite-base-3b-v1",
    "bench": "winogrande",
    "score": 0.6322,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "lab",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "redpajama-incite-base-3b-v1",
    "bench": "piqa",
    "score": 0.747,
    "source": "https://arxiv.org/html/2411.12372v1",
    "source_id": "redpajama-2024-original-zero-shot",
    "by": "lab",
    "date": "2024-11-19",
    "note": "Appendix C.2, table caption Results for RedPajama-INCITE-Base-3B-v1: zero-shot lm-evaluation-harness; LAMBADA OpenAI / WinoGrande / PIQA accuracy and HellaSwag acc_norm. Figure 2 identifies GPT-Neo as 2.7B and Pythia as the non-deduplicated step-143k rerun. No unlabeled GPT-Neo-size guess, deduplicated Pythia conflation, HELM output, or average imported."
  },
  {
    "model": "kandinsky-2-2",
    "bench": "kandinsky3-reference-text-win",
    "score": 0.32,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 10, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-kandinsky-2-2-text-win",
    "score": 0.52,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 10, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "kandinsky-2-2",
    "bench": "kandinsky3-reference-visual-win",
    "score": 0.48,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 10, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-kandinsky-2-2-visual-win",
    "score": 0.5,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 10, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "dall-e-3",
    "bench": "kandinsky3-reference-text-win",
    "score": 0.5,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "independent",
    "date": "2023-12-11",
    "note": "Figure 13, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-dall-e-3-text-win",
    "score": 0.36,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 13, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "dall-e-3",
    "bench": "kandinsky3-reference-visual-win",
    "score": 0.56,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "independent",
    "date": "2023-12-11",
    "note": "Figure 13, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-dall-e-3-visual-win",
    "score": 0.42,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 13, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "sdxl",
    "bench": "kandinsky3-reference-text-win",
    "score": 0.35,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "independent",
    "date": "2023-12-11",
    "note": "Figure 16, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-sdxl-text-win",
    "score": 0.53,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 16, overall text bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "sdxl",
    "bench": "kandinsky3-reference-visual-win",
    "score": 0.41,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "independent",
    "date": "2023-12-11",
    "note": "Figure 16, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted."
  },
  {
    "model": "kandinsky-3",
    "bench": "kandinsky3-vs-sdxl-visual-win",
    "score": 0.57,
    "source": "https://arxiv.org/html/2312.03511v2",
    "source_id": "kandinsky3-original-human-comparison",
    "by": "lab",
    "date": "2023-12-11",
    "note": "Figure 16, overall visual bar; printed outright-win fraction, not win+half-ties. 2,100 prompts / 21 categories, 12 judges per pairwise study, approximately 30,000 votes. Both/neither options present; rounded fractions are not forced to sum to one. SDXL refiner and DALL-E quality API settings are unspecified in report. No repeated per-category scores counted. This opponent-specific result remains raw evidence unless three genuinely measured models exist for that same comparator."
  },
  {
    "model": "kandinsky-2-1",
    "bench": "coco30k-fid-kandinsky21",
    "score": 8.21,
    "source": "https://huggingface.co/kandinsky-community/kandinsky-2-1/blob/04b366659663bf60deacb96fd3e38f1cb39a3be4/README.md",
    "source_id": "kandinsky21-card-original-fid",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Original card Evaluation table, own Kandinsky 2.1 row, FID (30k)=8.21. Zero-shot COCO-30k; generation/sampling details unspecified. Raw distance retained. Fixed latent unit 10 FID, sign reversed because lower is better; single-model source cannot calibrate. Date is review date, not generation date."
  },
  {
    "model": "melotts-english",
    "bench": "dailydialog-tts-whisper-large-wer",
    "score": 0.0639,
    "source": "https://arxiv.org/html/2601.20510v1",
    "source_id": "speech-tts-deepfake-age-ccby",
    "by": "independent",
    "date": "2026-01-28",
    "note": "Singh et al. Table 2 MeloTTS WER=0.0639, already a fraction. Section 3.1 specifies MeloTTS and English DailyDialog evaluation; 4,000 samples and Whisper-large + jiwer. This third original row was missing despite the source attribution already covering it. English checkpoint identity; speaker/backend seed unspecified."
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "gpqa-diamond",
    "score": 0.9539141414141414,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id LspviLnbeZ82WCngWp6rwR; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013780748450364692.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "gpqa-diamond",
    "score": 0.9558080808080808,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id QPk8jbJvo4986sbWtdte6J; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013710320714521202.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "gpqa-diamond",
    "score": 0.9046717171717171,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id 7F3QFMAkwnCV62HcvxP96q; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018537378631915503.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "gpqa-diamond",
    "score": 0.92739898989899,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id hxAhYJfXdPNanxatXpG5hC; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015505824295534454.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "gpqa-diamond",
    "score": 0.9431818181818182,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id aJD7ZfFftu2cZoToJhhzF7; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014938512008159717.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "gpqa-diamond",
    "score": 0.9059343434343434,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id 6gmbahPSxHf9xVhAYqpnhi; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019434397190192135.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id Di7y6sES4PGQbXNkXYhdqM; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id bSEaZLPehiNVDQL7HP9EJS; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "otis-mock-aime",
    "score": 0.9888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id 4uDz9b6SosxDSZf7swMD9D; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006674027922425511.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "otis-mock-aime",
    "score": 0.9805555555555555,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id Jhgwm7QpspGcvm7DUVqUDg; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014299486306926387.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id VZAUcNkzeMSMm9paxBhj6B; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "otis-mock-aime",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id LSfona7P3jCAnZFXJ2YA7F; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "frontiermath",
    "score": 0.9368421052631579,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id aBoQLaqnXsLPrsbN8wYgUp; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014434038295343787.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "frontiermath",
    "score": 0.887719298245614,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id 4tZWwV9tqEgsCaKeSuSoG4; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.018734030360524248.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "frontiermath",
    "score": 0.7894736842105263,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id iBQxysYR7zPxB6hxvJyTLc; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024191503146153773.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "frontiermath",
    "score": 0.5298245614035088,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id F4unBbW5M4VJogzCq4vGAb; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029616712041900365.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "frontiermath",
    "score": 0.8982456140350877,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id RHjHEPHEsWZqFuMjBTeC6r; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01793969072128637.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "frontiermath",
    "score": 0.9122807017543859,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id LPxhvBNEn7S3xkRZUgcvc9; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.016786209858442095.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "simpleqa-verified",
    "score": 0.739,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id dfT5VD7cQug8XW6GQHyJUF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013895037677965185.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "simpleqa-verified",
    "score": 0.465,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id GtJ5yd3ZPQ7HVcm6Q9ssQz; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015780495050030086.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "simpleqa-verified",
    "score": 0.414,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id HX9sRVWJtZiNvneALnVzs8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015583544104177553.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "simpleqa-verified",
    "score": 0.56,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id 6izDCw5ygCLZJEKPS8jV5T; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015704987954361718.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "simpleqa-verified",
    "score": 0.607,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id donMeEKBZZdqpQsojKWJEm; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.015452824654081525.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "simpleqa-verified",
    "score": 0.722,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id CgLfiGcNwMPVzxr8GdNUAK; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.014174516461485273.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "epoch-chess",
    "score": 0.61,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id d9TWVVwaH6vgp36FM7qGJ5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04902071300001973.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "epoch-chess",
    "score": 0.31,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id YwzVUtcUukgH4VxszaKNsi; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04648231987117317.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "epoch-chess",
    "score": 0.38,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id MpkdmmouJFUbKycL72HmSE; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.048783173121456344.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "epoch-ebr",
    "score": 0.5428571428571429,
    "source": "https://epoch.ai/data/benchmark_data.zip#ebr_bench.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file ebr_bench.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id LRUWNzFDcScwkTYuZYdeiV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05753831415997416.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "frontiermath-tier-4-v2",
    "score": 1,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id HgAUm5zxrWbkfYUdeJMGzL; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.8048780487804879,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id Xo7ecFEMqPJ4HicvJTvWZZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06265967111543964.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.5609756097560976,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id 7n3HBUK684GmdT6Tkycu6m; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07846686801046543.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.17073170731707318,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id Ymgj5VeEceBbzsLq6xEZRi; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05949419959829495.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.9,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id jwS3LYLCPbZpHDzzkX6LyV; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.051739520574625435.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.95,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id 9qKrTxPhpfW985gieKU56x; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.041175470771058854.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "epoch-mirrorcode",
    "score": 0.774,
    "source": "https://epoch.ai/data/benchmark_data.zip#mirrorcode.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file mirrorcode.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id mirrorcode-ml-2l-claude-opus-5-5; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0831.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "epoch-mystery-games",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id fQzkNRV69sKaJX7o9KKneQ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "epoch-mystery-games",
    "score": 0.71,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-opus-5-5_max'; run id fREi6AAmnfqUWkDqUhLLtw; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04560480215720687.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "epoch-mystery-games",
    "score": 0.65,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id 3PVdaSEYFJKgYUgHnF4DBu; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04793724854411022.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "epoch-mystery-games",
    "score": 0.07,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id bAERV8wdmoHMLz2K4bZCSK; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02564323999762428.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-7",
    "bench": "epoch-mystery-games",
    "score": 0.29,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'grok-4.7_xhigh'; run id Hk8bjr4u6KGqVE8SrPFKLq; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.045604802157206865.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-sol",
    "bench": "epoch-mystery-games",
    "score": 0.56,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-sol_max'; run id DnFTaE9qNsmBdYNqohhynY; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.049888765156985884.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "epoch-furniture",
    "score": 0.8,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6.1-sol_max'; run id R4vjsSfDrDo6ECQWGTUdWe; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04928872620664329.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "epoch-furniture",
    "score": 0.75,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-sonnet-5-5_max'; run id 5xgoV6NXxdDqscnFNpP2vv; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.05510647116121488.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-luna",
    "bench": "epoch-furniture",
    "score": 0.44166666666666665,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-09-28",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-luna_max'; run id GVtcaKADdbkuPoydqpKm3E; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06410142328406845. UPDATE: replaces existing epoch-internal row (score 0.2833333333333333, older run) - the earlier run is no longer in the file; newer run started 2026-09-28.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "frontiermath-erdos",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-fable-5-1_max'; run id bloom68-fable51-8t9up8nw9358oyrl; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. NEW internal file frontiermath_erdos.csv (benchmark_metadata.csv: 'FrontierMath-Erdos', in_eci False, released 2026-09-03); only 5 models in the file, mostly 0; needs the fragment added to the epoch-internal ledger entry and a new benchmark entry.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-6-astra",
    "bench": "frontiermath-erdos",
    "score": 0.029411764705882353,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-6-astra_max'; run id bloom68-ultima-e1dhs43ykrmakree; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0206414713705158. NEW internal file frontiermath_erdos.csv (benchmark_metadata.csv: 'FrontierMath-Erdos', in_eci False, released 2026-09-03); only 5 models in the file, mostly 0; needs the fragment added to the epoch-internal ledger entry and a new benchmark entry.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "frontiermath-erdos",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'claude-fable-5_max'; run id bloom68-fable-vn2l2xoe9902k9rt; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. NEW internal file frontiermath_erdos.csv (benchmark_metadata.csv: 'FrontierMath-Erdos', in_eci False, released 2026-09-03); only 5 models in the file, mostly 0; needs the fragment added to the epoch-internal ledger entry and a new benchmark entry.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "frontiermath-erdos",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-07 (zip sha256 8802ec0803986ce0131006b78f4e6658257c8975828ef96ec32633bc360642d4, ETag \"7f1ab9da8b81b22fd925f10eacc6a7fb\"); exact model version 'gpt-5.6-sol_max'; run id bloom68-sol-jx4yy8n4naukwwge; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. NEW internal file frontiermath_erdos.csv (benchmark_metadata.csv: 'FrontierMath-Erdos', in_eci False, released 2026-09-03); only 5 models in the file, mostly 0; needs the fragment added to the epoch-internal ledger entry and a new benchmark entry.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-fable-5",
    "bench": "lmarena-text",
    "score": 1504.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'claude-fable-5-high'; 38,387 votes; published 95% interval 1499.9–1508.7. REFRESH: existing row (1505.7) cites older revision 6931a5ef entry 'claude-fable-5'; that entry no longer exists in the latest split and the model now appears as 'claude-fable-5-high' (webdev and agent rows already use the High entry). File is byte-identical (same LFS oid) at the newest revision be06b33784531833af86ad96817e12ba2a965b30.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "claude-fable-5",
    "bench": "lmarena-vision",
    "score": 1308.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'claude-fable-5-high'; 13,709 votes; published 95% interval 1301.5–1316.0. REFRESH: existing row (1309.5) cites older revision 6931a5ef entry 'claude-fable-5'; that entry no longer exists in the latest split and the model now appears as 'claude-fable-5-high' (webdev and agent rows already use the High entry). File is byte-identical (same LFS oid) at the newest revision be06b33784531833af86ad96817e12ba2a965b30.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.885965,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (codex harness (configs/models/openai/gpt-6-astra.yaml: model gpt-6-astra, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 2-2 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.877193,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'Claude-Fable-5.1 (high)', model_config 'anthropic/fable_51_high' (Claude Code harness (configs/models/anthropic/fable_51_high.yaml: model claude-fable-5-1, adaptive thinking; the max config was not run on this set), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 2-2 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.578947,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (Qwen Code harness via OpenRouter (configs/models/qwen/qwen3.8_max.yaml: qwen/qwen3.8-max, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 1-1 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.45614,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'Muse Spark 1.3', model_config 'meta/muse_spark_13' (Muse harness (configs/models/meta/muse_spark_13.yaml: muse-spark-1.3, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 1-2 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.403509,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (Antigravity CLI harness (configs/models/gemini/gemini-38-flash.yaml: gemini-3.8-flash, reasoning_effort high), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 1-2 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-arxivmath-2026-08",
    "score": 0.385965,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0826_outputs/blob/64731d80cf7dd2a83d842c4deb9739c81d129efa/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena ArXivMath August 2026 outputs dataset (HF revision 64731d80cf7dd2a83d842c4deb9739c81d129efa); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (Kimi CLI harness via OpenRouter (configs/models/moonshot/k3.yaml: moonshotai/kimi-k3, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 57/57 problems, 1-1 attempts per problem; score = mean final-answer correctness as published in the 'correct' column (graded by MathArena answer_judge), averaged per problem then over problems. August protocol differs from earlier months: open-problem prompt, allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt), LLM answer judge (configs/competitions/arxiv/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.819444,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (codex harness (configs/models/openai/gpt-6-astra.yaml: model gpt-6-astra, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 3-3 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.797619,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'Claude-Fable-5.1 (high)', model_config 'anthropic/fable_51_high' (Claude Code harness (configs/models/anthropic/fable_51_high.yaml: model claude-fable-5-1, adaptive thinking; the max config was not run on this set), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 1-1 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.696429,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'Qwen3.8-Max', model_config 'qwen/qwen3.8_max' (Qwen Code harness via OpenRouter (configs/models/qwen/qwen3.8_max.yaml: qwen/qwen3.8-max, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 1-1 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "kimi-k3",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.619048,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'Kimi K3 (Think)', model_config 'moonshot/k3' (Kimi CLI harness via OpenRouter (configs/models/moonshot/k3.yaml: moonshotai/kimi-k3, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 1-1 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.27381,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (Antigravity CLI harness (configs/models/gemini/gemini-38-flash.yaml: gemini-3.8-flash, reasoning_effort high), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 3-3 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "matharena-brokenarxiv-2026-08",
    "score": 0.202381,
    "source": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826_outputs/blob/7da551be2a80d7721bd6ee77ef14c8894fdef686/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-15",
    "note": "MathArena BrokenArXiv August 2026 outputs dataset (HF revision 7da551be2a80d7721bd6ee77ef14c8894fdef686); published model_name 'Muse Spark 1.3', model_config 'meta/muse_spark_13' (Muse harness (configs/models/meta/muse_spark_13.yaml: muse-spark-1.3, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 56/56 problems, 1-1 attempts per problem; score = mean normalized judge score from the 'correct' column (judge: arxiv_judge_gemini_38_flash; 1 = explicitly recognises the statement is false), averaged per problem then over problems. August protocol differs from earlier months: allow_harness: true (shell, Python, SageMath, no internet; 12 h / $100 per attempt); judge changed to Gemini 3.8 Flash (configs/competitions/arxiv_false/august.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gpt-6-astra",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.652174,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'GPT-6 Astra (max)', model_config 'openai/gpt-6-astra' (codex harness (configs/models/openai/gpt-6-astra.yaml: model gpt-6-astra, reasoning_effort max), eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "claude-opus-5",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.326087,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Claude-Opus-5 (max)', model_config 'anthropic/opus_5_max_128' (config file not present in eth-sri/matharena at b89f2f0ad64c; identity from published model_name only, eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.217391,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Gemini 3.7 Flash', model_config 'gemini/gemini-37-flash' (plain API with MathArena Lean tools (configs/models/gemini/gemini-37-flash.yaml: gemini-3.7-flash, reasoning_effort high, no harness), eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.195652,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Gemini 3.8 Flash', model_config 'gemini/gemini-38-flash' (Antigravity CLI harness (configs/models/gemini/gemini-38-flash.yaml: gemini-3.8-flash, reasoning_effort high), eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "claude-fable-5",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.152174,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Claude-Fable-5 (max)', model_config 'anthropic/fable_5_max_128' (config file not present in eth-sri/matharena at b89f2f0ad64c; identity from published model_name only, eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.985667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2957/3000 correct (noAnswer=6). Roblox LLM Leaderboard entry 'Claude Opus 5.5 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-5-5_mc_effort-medium.json5 / claude-opus-5-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.870192,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 44+84+53=181/208. Roblox LLM Leaderboard entry 'Claude Opus 5.5 (Medium)' (https://www.anthropic.com/claude/opus); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-opus-5-5_mc_effort-medium.json5 / claude-opus-5-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "robloxqa-v2-mc",
    "score": 0.966333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2899/3000 correct (noAnswer=2). Roblox LLM Leaderboard entry 'Claude Sonnet 5.5 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-sonnet-5-5_mc_effort-medium.json5 / claude-sonnet-5-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "luau-leetcode-full-pass",
    "score": 0.807692,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 44+80+44=168/208. Roblox LLM Leaderboard entry 'Claude Sonnet 5.5 (Medium)' (https://www.anthropic.com/claude/sonnet); effort/setting as labelled 'Medium' (the only setting listed for this model); run files claude-sonnet-5-5_mc_effort-medium.json5 / claude-sonnet-5-5_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-astra",
    "bench": "robloxqa-v2-mc",
    "score": 0.985,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2955/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 6 Astra (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-astra); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-astra_mc_effort-medium.json5 / gpt-6-astra_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-astra",
    "bench": "luau-leetcode-full-pass",
    "score": 0.870192,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 45+83+53=181/208. Roblox LLM Leaderboard entry 'GPT 6 Astra (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-astra); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-astra_mc_effort-medium.json5 / gpt-6-astra_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-luna",
    "bench": "robloxqa-v2-mc",
    "score": 0.956333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2869/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 6 Luna (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-luna); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-luna_mc_effort-medium.json5 / gpt-6-luna_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-luna",
    "bench": "luau-leetcode-full-pass",
    "score": 0.807692,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 44+81+43=168/208. Roblox LLM Leaderboard entry 'GPT 6 Luna (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-luna); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-luna_mc_effort-medium.json5 / gpt-6-luna_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-sol",
    "bench": "robloxqa-v2-mc",
    "score": 0.978333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2935/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 6 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-sol_mc_effort-medium.json5 / gpt-6-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-sol",
    "bench": "luau-leetcode-full-pass",
    "score": 0.826923,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 42+80+50=172/208. Roblox LLM Leaderboard entry 'GPT 6 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-6-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6-sol_mc_effort-medium.json5 / gpt-6-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "robloxqa-v2-mc",
    "score": 0.985667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2957/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'GPT 6.1 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-6.1-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6.1-sol_mc_effort-medium.json5 / gpt-6.1-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "luau-leetcode-full-pass",
    "score": 0.870192,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 45+84+52=181/208. Roblox LLM Leaderboard entry 'GPT 6.1 Sol (Medium)' (https://developers.openai.com/api/docs/models/gpt-6.1-sol); effort/setting as labelled 'Medium' (the only setting listed for this model); run files gpt-6.1-sol_mc_effort-medium.json5 / gpt-6.1-sol_effort-medium.json5. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "mideind-icelandic-average",
    "score": 0.681688,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'GPT-6.1 Sol' (openai/gpt-6.1-sol), thinking='high', gen_kwargs={\"max_completion_tokens\": 32000, \"reasoning_effort\": \"high\", \"allowed_openai_params\": [\"reasoning_effort\"]}; highest of the listed settings ['high', 'low']. Published 'average' (Knowledge 0.502082, Language Proficiency 0.816392). Date = latest benchmark_evaluated_at. Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "mideind-icelandic-average",
    "score": 0.621202,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Claude Sonnet 5.5' (anthropic/claude-sonnet-5-5), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"reasoning_effort\": \"high\"}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.425866, Language Proficiency 0.767705). Date = latest benchmark_evaluated_at. Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Ledger: source_id exists but its match list pins the older revision; add this revision URL to the match list before import.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "claude-fable-5",
    "bench": "terminal-bench-4",
    "score": 0.4455,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-anthropic-claude-fable-5-max-claude-code.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-fable-5' (display 'Fable 5'), agent Claude Code (claude-code 2.1.231), reasoning_effort max. metrics.accuracy 44.55% ± 3.85 (95% CI half-width); 147/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-06-09 is the model release date. file sha256 487f60477db9c36bd5cdb3d50c80634233bfa53de3368cf33337a771f17aa07e.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-opus-5",
    "bench": "terminal-bench-4",
    "score": 0.5182,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-anthropic-claude-opus-5-max-claude-code.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-opus-5' (display 'Opus 5'), agent Claude Code (claude-code 2.1.231), reasoning_effort max. metrics.accuracy 51.82% ± 3.39 (95% CI half-width); 171/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-07-24 is the model release date. file sha256 06fbebf9bf42c430c49b020f86b9365f5ae772c6063bab4fce98f54e349ed80c.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "glm-5-3",
    "bench": "terminal-bench-4",
    "score": 0.4182,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-anthropic-glm-5-3-max-claude-code.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/glm-5.3' (display 'GLM-5.3'), agent Claude Code (claude-code 2.1.207), reasoning_effort max. metrics.accuracy 41.82% ± 3.23 (95% CI half-width); 138/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-08-14 is the model release date. file sha256 f8c5a8ef9dab054e78444f6e2e4b756dda2cf193839a3a357ce2a4d32a0c24c9.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "terminal-bench-4",
    "score": 0.1727,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-openai-gpt-5-6-luna-max-codex.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-5.6-luna' (display 'GPT-5.6 Luna'), agent Codex (codex 0.149.1), reasoning_effort max. metrics.accuracy 17.27% ± 2.85 (95% CI half-width); 57/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-06-26 is the model release date. file sha256 59ae3614eca2b0c1f087b7a50d65e3fc40dd2ef0188c2fab40d32c5a50f823a7.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "terminal-bench-4",
    "score": 0.3727,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-openai-gpt-5-6-sol-max-codex.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-5.6-sol' (display 'GPT-5.6 Sol'), agent Codex (codex 0.149.1), reasoning_effort max. metrics.accuracy 37.27% ± 3.78 (95% CI half-width); 123/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-06-26 is the model release date. file sha256 b41e5f5f15aeb7657f9d32b33a868aecab6dd6db2d48b7480638455664102c5b.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "terminal-bench-4",
    "score": 0.2152,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-openai-gpt-5-6-terra-max-codex.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-5.6-terra' (display 'GPT-5.6 Terra'), agent Codex (codex 0.149.1), reasoning_effort max. metrics.accuracy 21.52% ± 3.25 (95% CI half-width); 71/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-06-26 is the model release date. file sha256 1c7ca98885e40ed4dc31d23391982848b2c1dd87b0595743fa242ad848f1b670.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "grok-4-5",
    "bench": "terminal-bench-4",
    "score": 0.1242,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-xai-grok-4-5-none-grok-build.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'xai/grok-4.5' (display 'Grok 4.5'), agent Grok Build (grok-build 1.0.5), reasoning_effort None. metrics.accuracy 12.42% ± 2.62 (95% CI half-width); 41/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-07-16 is the model release date. file sha256 fa618ecb274776479019af6d62c538197cbb563a44d8b8a45752dde51d603b93.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "grok-4-6",
    "bench": "terminal-bench-4",
    "score": 0.203,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-xai-grok-4-6-none-grok-build.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'xai/grok-4.6' (display 'Grok 4.6'), agent Grok Build (grok-build 1.0.5), reasoning_effort None. metrics.accuracy 20.3% ± 3.09 (95% CI half-width); 67/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-08-12 is the model release date. file sha256 31526e5dff0e06680085bb5029700b0fddc607962e7c6aea9a911600c6b2df1a.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "terminal-bench-4",
    "score": 0.1909,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json",
    "by": "independent",
    "date": "2026-09-01",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'gemini/gemini-3.8-flash' (display 'Gemini 3.8 Flash'), agent mini-SWE-agent (mini-swe-agent 2.4.6), reasoning_effort high. metrics.accuracy 19.09% ± 3.36 (95% CI half-width); 63/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-02 is the model release date. file sha256 835f2090e7d40d9e5c6c5483da49c5f3147a0304e615df49c71aafa859a62c50.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "terminal-bench-4",
    "score": 0.5788,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-fable-5-1' (display 'Fable 5.1'), agent Claude Code (claude-code 2.1.257), reasoning_effort max. metrics.accuracy 57.88% ± 3.76 (95% CI half-width); 191/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-01 is the model release date. file sha256 cccdb936c1266db66ce922db48144ed0716c1294eec3dcc1bdcf976711c92813.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "terminal-bench-4",
    "score": 0.1121,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json",
    "by": "independent",
    "date": "2026-09-02",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'gemini/gemini-3.7-flash' (display 'Gemini 3.7 Flash'), agent mini-SWE-agent (mini-swe-agent 2.4.6), reasoning_effort high. metrics.accuracy 11.21% ± 2.45 (95% CI half-width); 37/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-08-13 is the model release date. file sha256 013ace9d0c06b97aed4227b624aff0724257b2ca44a113553a03707545efc0cc.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "terminal-bench-4",
    "score": 0.6485,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-anthropic-claude-opus-5-5-max-claude-code.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-opus-5-5' (display 'Opus 5.5'), agent Claude Code (claude-code 2.1.284), reasoning_effort max. metrics.accuracy 64.85% ± 3.11 (95% CI half-width); 214/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-22 is the model release date. file sha256 782d7a4b5e832873b660fad00da9a8d2eb2f37b6bf3a7e65e4f6ffa7cd82a53c.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "terminal-bench-4",
    "score": 0.6182,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-anthropic-claude-sonnet-5-5-max-claude-code.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-sonnet-5-5' (display 'Sonnet 5.5'), agent Claude Code (claude-code 2.1.284), reasoning_effort max. metrics.accuracy 61.82% ± 2.94 (95% CI half-width); 204/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-28 is the model release date. file sha256 723a52bf4c657d3bde65ce61a4dd5caffbae3aa2359f9944181bb5c1eb81455b.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "terminal-bench-4",
    "score": 0.1455,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-meta-muse-spark-1-3-xhigh-muse-code.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'meta/muse-spark-1.3' (display 'Muse Spark 1.3'), agent Muse Code (muse-code 1.4.1), reasoning_effort xhigh. metrics.accuracy 14.55% ± 2.94 (95% CI half-width); 48/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-02 is the model release date. file sha256 0f63ac5aafcaf947308057eb937a5a1901331948a83cf84645f99117bb1542ff.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "terminal-bench-4",
    "score": 0.5818,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-openai-gpt-6-1-sol-max-codex.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-6.1-sol' (display 'GPT-6.1 Sol'), agent Codex (codex 0.159.0), reasoning_effort max. metrics.accuracy 58.18% ± 3.11 (95% CI half-width); 192/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-29 is the model release date. file sha256 2bb4407be33d9ea12c9d1173ee749b1296bba0b5fb8442c87c9cc7be3c2d8bf7.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-6-luna",
    "bench": "terminal-bench-4",
    "score": 0.1636,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-openai-gpt-6-luna-max-codex.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-6-luna' (display 'GPT-6 Luna'), agent Codex (codex 0.159.0), reasoning_effort max. metrics.accuracy 16.36% ± 2.72 (95% CI half-width); 54/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-22 is the model release date. file sha256 f063aee31d6599b95ebe37341071bc625bf042362b45590db4cc50efbb5cb8a8.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "gpt-6-sol",
    "bench": "terminal-bench-4",
    "score": 0.4939,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-09-30-openai-gpt-6-sol-max-codex.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'openai/gpt-6-sol' (display 'GPT-6 Sol'), agent Codex (codex 0.159.0), reasoning_effort max. metrics.accuracy 49.39% ± 3.23 (95% CI half-width); 163/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-22 is the model release date. file sha256 c516de23e47a1d39f10844f10c8d387a38febb9d93f0ab6b069507edcb84c0cd.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "terminal-bench-4",
    "score": 0.2697,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-10-06-qwen-qwen3-8-max-0902-max-claude-code.json",
    "by": "independent",
    "date": "2026-10-06",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'qwen/qwen3.8-max-0902' (display 'Qwen3.8-Max-0902'), agent Claude Code (claude-code 2.1.285), reasoning_effort max. metrics.accuracy 26.97% ± 3.78 (95% CI half-width); 89/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-09-02 is the model release date. file sha256 96e243adaf488b3c8414821579eff6a54b664e673d5fcc532dbe0a48afb03acf.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-opus-5",
    "bench": "automationbench-public-zapier",
    "score": 0.503,
    "source": "https://github.com/zapier/AutomationBench/blob/4a8e1061254004d9dac807054eed33fad7d1ff14/README.md",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Zapier AutomationBench README table 'Pass rates on the 600-task public benchmark' (benchmark maintainer's runs, not the official private-set leaderboard at zapier.com/benchmarks); row 'Claude Opus 5', reasoning effort max, 50.3%. README notes Fable 5 score is after the July 2026 stricter classifier; package version for each row not stated (v1.0.6 released in the commit before), so kept separate from automationbench / automationbench-public / automationbench-v1-0-6. Date = README commit date.",
    "source_id": "zapier-automationbench-readme"
  },
  {
    "model": "kimi-k3",
    "bench": "automationbench-public-zapier",
    "score": 0.4667,
    "source": "https://github.com/zapier/AutomationBench/blob/4a8e1061254004d9dac807054eed33fad7d1ff14/README.md",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Zapier AutomationBench README table 'Pass rates on the 600-task public benchmark' (benchmark maintainer's runs, not the official private-set leaderboard at zapier.com/benchmarks); row 'Kimi K3', reasoning effort max, 46.67%. README notes Fable 5 score is after the July 2026 stricter classifier; package version for each row not stated (v1.0.6 released in the commit before), so kept separate from automationbench / automationbench-public / automationbench-v1-0-6. Date = README commit date.",
    "source_id": "zapier-automationbench-readme"
  },
  {
    "model": "claude-fable-5",
    "bench": "automationbench-public-zapier",
    "score": 0.4617,
    "source": "https://github.com/zapier/AutomationBench/blob/4a8e1061254004d9dac807054eed33fad7d1ff14/README.md",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Zapier AutomationBench README table 'Pass rates on the 600-task public benchmark' (benchmark maintainer's runs, not the official private-set leaderboard at zapier.com/benchmarks); row 'Claude Fable 5', reasoning effort max, 46.17%. README notes Fable 5 score is after the July 2026 stricter classifier; package version for each row not stated (v1.0.6 released in the commit before), so kept separate from automationbench / automationbench-public / automationbench-v1-0-6. Date = README commit date.",
    "source_id": "zapier-automationbench-readme"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "automationbench-public-zapier",
    "score": 0.4583,
    "source": "https://github.com/zapier/AutomationBench/blob/4a8e1061254004d9dac807054eed33fad7d1ff14/README.md",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Zapier AutomationBench README table 'Pass rates on the 600-task public benchmark' (benchmark maintainer's runs, not the official private-set leaderboard at zapier.com/benchmarks); row 'GPT-5.6 Sol', reasoning effort max, 45.83%. README notes Fable 5 score is after the July 2026 stricter classifier; package version for each row not stated (v1.0.6 released in the commit before), so kept separate from automationbench / automationbench-public / automationbench-v1-0-6. Date = README commit date.",
    "source_id": "zapier-automationbench-readme"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "automationbench-public-zapier",
    "score": 0.3717,
    "source": "https://github.com/zapier/AutomationBench/blob/4a8e1061254004d9dac807054eed33fad7d1ff14/README.md",
    "by": "independent",
    "date": "2026-08-04",
    "note": "Zapier AutomationBench README table 'Pass rates on the 600-task public benchmark' (benchmark maintainer's runs, not the official private-set leaderboard at zapier.com/benchmarks); row 'GPT-5.6 Terra', reasoning effort max, 37.17%. README notes Fable 5 score is after the July 2026 stricter classifier; package version for each row not stated (v1.0.6 released in the commit before), so kept separate from automationbench / automationbench-public / automationbench-v1-0-6. Date = README commit date.",
    "source_id": "zapier-automationbench-readme"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.899,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Claude Opus 5.5' = 89.9 (§8.2: 89.9%, average over five trials). Public/commercial split not stated. Standard configuration: adaptive thinking at max effort, default sampling, averaged over five trials. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "swe-bench-multilingual",
    "score": 0.939,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A, row 'SWE-bench Multilingual', column 'Claude Opus 5.5' = 93.9 (§8.2: 300 problems, nine languages, average over five trials). Adaptive thinking at max effort, default sampling. Scaffold not stated. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "terminal-bench-4",
    "score": 0.664,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A, row 'Terminal-Bench 4.0', column 'Claude Opus 5.5' = 66.4 (§8.5, p.178: 66.36%). xhigh effort (the card says max effort scored 64.8%, within noise). Claude Code in --bare mode, five trials per task (330 trials), SE ±2.6 pts. Safeguards on: flagged requests answered by a fallback model (2.5% of requests, 10% of trials). Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "hle",
    "score": 0.644,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam', sub-row 'No tools', column 'Claude Opus 5.5' = 64.4. §8.11.1 (p.184): full 2,500-question multimodal set, reasoning-only with no tools, thinking set to auto, 1M total-token cap, no compaction, graded by Claude Opus 4.6. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "hle-with-tools",
    "score": 0.677,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam', sub-row 'With tools', column 'Claude Opus 5.5' = 67.7. §8.11.1 (p.184): full 2,500-question set with web search, web fetch, programmatic tool calling and code execution. Thinking auto, 1M total-token cap, no compaction, Claude Opus 4.6 grader. Sources that discuss HLE are blocklisted and flagged transcripts are re-graded. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "hle-with-tools",
    "score": 0.656,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam', sub-row 'With tools', column 'Claude Fable 5.1' = 65.6. Comparison column for an earlier Anthropic model. This differs from the 65.0% on the Fable 5.1 announcement page. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "deepswe-v1-1",
    "score": 0.742,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=175",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.3 'DeepSWE v1.1' prose: 'scored an average of 74.2% over five trials' on 113 tasks. Harness not stated. Standard configuration: adaptive thinking at max effort. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "toolathlon-verified",
    "score": 0.778,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=211",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, Table 8.14.5.A 'Toolathlon scores (internal harness)', row 'Claude Opus 5.5', column 'Pass@1' = 77.8. Toolathlon-Verified (June 2026 release), 108 tasks, Pass@1 averaged over three trials. Adaptive thinking, max effort, safety classifiers on. One classifier stop and six sandbox-monitor halts are counted as failures. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "healthbench",
    "score": 0.681,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=213",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.15.1 'HealthBench results' prose: raw score 68.1%. The length-adjusted 60.6% is not used. Adaptive thinking, max effort, Claude Opus 4.8 grader, five trials, no tools or custom system prompt. Safety classifiers on with refusal fallback to Claude Opus 5. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "global-mmlu",
    "score": 0.943,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=215",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.16.1 'GMMLU results' prose: average accuracy 94.3% across 42 languages (Global MMLU, full). Adaptive thinking, max effort, single trial, no tools or custom system prompt. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "global-mmlu",
    "score": 0.892,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=215",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.16.1 'GMMLU results' prose: Claude Sonnet 5 89.2% average across 42 languages. Adaptive thinking, max effort, single trial, no tools. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "chartography-with-tools",
    "score": 0.89,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.13.1 'Chartography' prose: 'Claude Opus 5.5 achieved a score of 64.4% without tools and 89.0% with tools'. The with-tools figure is used: a container with the image file, standard libraries and an image-cropping tool. Adaptive thinking, max effort, five runs. Anthropic's internal grading implementation, with Gemini 3.5 Flash as grader (following the Surge AI leaderboard). Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "chartography-with-tools",
    "score": 0.884,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.13.1 'Chartography' prose: 'Claude Fable 5.1 scored 44.8% and 88.4%' (without and with tools). With-tools figure. Container plus crop tool, adaptive thinking, max effort, five runs, Gemini 3.5 Flash grader. The card says earlier cards used a Claude Sonnet 4.6 judge and showed slightly lower scores. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "chartography-with-tools",
    "score": 0.834,
    "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-22",
    "note": "Anthropic, Claude Opus 5.5 System Card, §8.13.1 'Chartography' prose: 'Claude Opus 5 achieved scores of 29.8% and 83.4%' (without and with tools). With-tools figure. Container plus crop tool, adaptive thinking, max effort, five runs, Gemini 3.5 Flash grader. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "terminal-bench-4",
    "score": 0.103,
    "source": "https://www.anthropic.com/claude-sonnet-5-5",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic announcement 'Claude Sonnet 5.5', benchmark table, row 'Agentic coding Terminal-Bench 4.0', column 'Sonnet 5' = 10.3%. Also stated in the text: 'compared to Sonnet 5's 10.3%'. Effort not stated. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.812,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Claude Fable 5.1/Mythos 5.1' = 81.2 (§8.2: 'Fable 5.1 achieved 81.2%', average over five trials). Public/commercial split not stated. Standard configuration: adaptive thinking at max effort, default sampling, averaged over five trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "swe-bench-multilingual",
    "score": 0.891,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'SWE-bench Multilingual', column 'Claude Fable 5.1/Mythos 5.1' = 89.1 (§8.2: 300 problems, nine languages, average over five trials). Adaptive thinking at max effort; scaffold not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "deepswe-v1-1",
    "score": 0.674,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=168",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.3 'DeepSWE v1.1' (113 tasks): 'Fable 5.1 scored an average of 67.4% over five trials'. Harness not stated; standard configuration adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "terminal-bench-4",
    "score": 0.558,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=171",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.6 'Terminal-Bench 4.0' (66 tasks): 'Claude Fable 5.1 scored 55.8%' (Table 8.1.A rounds to 56%). 15 trials per task (990 trials), standard error ±1.6-2 pts; Claude Code in --bare mode, maximum thinking effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "terminal-bench-4",
    "score": 0.42,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=171",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.6 'Terminal-Bench 4.0': 'Claude Fable 5 42.0%' (Table 8.1.A: 42%). 15 trials per task (990 trials); Claude Code --bare mode, maximum thinking effort; public leaderboard reports Fable 5 at 44.5% (five trials); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "terminal-bench-4",
    "score": 0.523,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=171",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.6 'Terminal-Bench 4.0': 'Claude Opus 5 scored 52.3%' (Table 8.1.A: 52%). 15 trials per task (990 trials); Claude Code --bare mode, maximum thinking effort; public leaderboard reports Opus 5 at 51.8%; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "hle",
    "score": 0.609,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Claude Fable 5.1/Mythos 5.1' = 60.9%. §8.12.1: full 2,500-question multimodal set, reasoning-only, thinking auto, 1M total-token cap, no compaction, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "hle",
    "score": 0.578,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Claude Fable 5/Mythos 5' = 57.8%. Full 2,500-question set; configuration in §8.12.1 is described for Fable 5.1; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "hle",
    "score": 0.566,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Claude Opus 5' = 56.6%. Full 2,500-question set; configuration in §8.12.1 is described for Fable 5.1; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "hle-with-tools",
    "score": 0.65,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Claude Fable 5.1/Mythos 5.1' = 65.0%. §8.12.1: full 2,500-question set; web search, restricted web fetch, programmatic tool calling, code execution; thinking auto; 1M total-token cap; no compaction; HLE sources blocklisted; Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "hle-with-tools",
    "score": 0.638,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Claude Fable 5/Mythos 5' = 63.8%. Full 2,500-question set; tool configuration in §8.12.1 is described for Fable 5.1; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "hle-with-tools",
    "score": 0.636,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=167",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Claude Opus 5' = 63.6%. Full 2,500-question set; tool configuration in §8.12.1 is described for Fable 5.1; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "chartography-with-tools",
    "score": 0.862,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=184",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.14.1 'Chartography' (Surge AI, 100 tasks): 'Claude Fable 5.1 achieved ... 86.2% with tools'. Container with the image and standard libraries plus an image-cropping tool; adaptive thinking, max effort; averaged over five runs; Anthropic's internal grading implementation (grader not named here; the Opus 5.5 card says earlier cards used Claude Sonnet 4.6 as judge for Claude models); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "chartography-with-tools",
    "score": 0.842,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=184",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.14.1 'Chartography': 'Claude Fable 5 scored 36.6% and 84.2%' (no tools / with tools). With-tools figure; container + image-cropping tool; adaptive thinking, max effort; five runs; internal grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "toolathlon-verified",
    "score": 0.778,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=194",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, Table 8.15.5.A 'Toolathlon scores (internal harness)', row 'Claude Fable 5.1', column 'Pass@1' = 77.8. Toolathlon-Verified (authors' June 2026 final release), 108 tasks, Pass@1 averaged over three trials; adaptive thinking, max effort; production configuration with safety classifiers and Claude Opus 4.8 refusal fallback (11 of 324 trials used the fallback; 4 terminated trials counted as failures); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "healthbench",
    "score": 0.667,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=198",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.17.1 'HealthBench results': 'Claude Fable 5.1 achieved a raw score of 66.7%' (raw score, not the 60% length-adjusted figure). Full HealthBench (5,000 conversations); adaptive thinking, max effort; Claude Opus 4.8 grader; five trials; no tools or custom system prompt; safety classifiers active with refusal fallback to Claude Opus 5; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "healthbench",
    "score": 0.612,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=198",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.17.1 'HealthBench results': raw score 'Claude Fable 5 at 61.2%'. Adaptive thinking, max effort; Claude Opus 4.8 grader; five trials; no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5-1",
    "bench": "global-mmlu",
    "score": 0.94,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.18.1 'GMMLU results': 'Claude Fable 5.1 achieved an average accuracy of 94.0% across 42 languages'. Adaptive thinking, max effort; single trial; no tools or custom system prompt; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "global-mmlu",
    "score": 0.936,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.18.1 'GMMLU results': 'Claude Fable 5 at 93.6%' (average across 42 languages). Adaptive thinking, max effort; single trial; no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "global-mmlu",
    "score": 0.925,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.18.1 'GMMLU results': 'Claude Opus 5 at 92.5%' (average across 42 languages). Adaptive thinking, max effort; single trial; no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "global-mmlu",
    "score": 0.89,
    "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20&%20Claude%20Mythos%205.1%20System%20Card.pdf#page=200",
    "by": "lab",
    "date": "2026-09-01",
    "note": "Anthropic Claude Fable 5.1 & Mythos 5.1 System Card, §8.18.1 'GMMLU results': 'Claude Sonnet 5 at 89.0%' (average across 42 languages; the later Opus 5.5 card gives Sonnet 5 89.2%). Adaptive thinking, max effort; single trial; no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.8,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card (June 9, 2026; revised June 25, 2026), Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Fable 5' = 80 (§8.2: 'Claude Fable 5 achieved 80%', average over 5 trials). Split not stated; adaptive thinking, max effort; Fable scores reflect production safeguards incl. fallback to Opus 4.8; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.692,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'SWE-bench Pro', column 'Opus 4.8' = 69.2. Split not stated; settings for Opus 4.8 not restated in this card; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "swe-bench-verified",
    "score": 0.95,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'SWE-bench Verified', column 'Fable 5' = 95 (§8.2: 500-problem subset, 'Fable 5 achieved 95%', average over 5 trials). Standard configuration, thinking blocks included; production safeguards incl. fallback to Opus 4.8; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "swe-bench-verified",
    "score": 0.886,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'SWE-bench Verified', column 'Opus 4.8' = 88.6; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "terminal-bench-2.1",
    "score": 0.843,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=254",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, §8.3 'Terminal-Bench 2.1' (Table 8.1.A also 84.3): 'Claude Fable 5: achieved 84.3% mean reward'. mini-SWE-agent harness on GKE, 1x timeout, 3x memory ceiling; high effort; 20.9% of trials hit a safety refusal and fell back to Claude Opus 4.8 for the rest of the trajectory; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "terminal-bench-2.1",
    "score": 0.827,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=255",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, §8.3 'Terminal-Bench 2.1' (Table 8.1.A also 82.7): 'Claude Opus 4.8: achieved 82.7% mean reward, averaged over 5 attempts for each one of the 89 unique tasks (445 trials), at high effort'. mini-SWE-agent harness on GKE; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "hle",
    "score": 0.498,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Opus 4.8' = 49.8. §8.14.1 describes the full 2,500-question set, reasoning-only, thinking auto, 1M token cap, Opus 4.6 grader (stated for Mythos 5); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "hle-with-tools",
    "score": 0.579,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Opus 4.8' = 57.9. §8.14.1: web search, web fetch, programmatic tool calling, code execution; HLE sources blocklisted (stated for Mythos 5); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "charxiv-rq",
    "score": 0.805,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'CharXiv Reasoning' / 'No tools', column 'Opus 4.8' = 80.5. §8.16.8: 1,000 validation questions, reference prompts but Claude Sonnet 4.6 as grader instead of GPT-4o, adaptive thinking max effort, five runs (stated for Mythos 5); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "osworld-verified",
    "score": 0.85,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'OSWorld-Verified', column 'Fable 5' = 85.0. §8.16.3 / Figure 8.16.3.A: 361 tasks, 1080p, 100 steps, auto-thinking at max effort, pass@1 averaged over five runs; zoom-tool fix and 128K per-turn token limit; Fable scores reflect production safeguards; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "osworld-verified",
    "score": 0.834,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A, row 'OSWorld-Verified', column 'Opus 4.8' = 83.4. 361 tasks, 100 steps, auto-thinking at max effort, pass@1 averaged over five runs (Figure 8.16.3.A); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "usamo-2026",
    "score": 0.967,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=260",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, §8.10 'USAMO 2026': 'Under similar settings, Opus 4.8 scored 96.7%'. MathArena grading methodology: proofs rewritten by Gemini 3.1 Pro and judged by a 3-model panel (Gemini 3.1 Pro, Claude Opus 4.6, Claude Mythos Preview), minimum score taken; 10 attempts per problem (as stated for Mythos 5); Opus 4.8 effort not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "usamo-2026",
    "score": 0.693,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=260",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, §8.10 'USAMO 2026': 'Under similar settings ... Opus 4.7 scored 69.3%'. MathArena grading methodology, 3-judge panel, minimum score; 10 attempts per problem (as stated for Mythos 5); effort not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "toolathlon",
    "score": 0.617,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=295",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.17.8.A 'Toolathlon scores (internal harness)', row 'Claude Fable 5', column 'Pass@1' = 61.7. Original Toolathlon (upstream tasks, 108 tasks; not the later Verified release), Pass@1 averaged over 3 trials; adaptive thinking at max effort; pinned data feeds/containers; card says its scores run ~3 points above a strictly upstream-equivalent harness; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "toolathlon",
    "score": 0.599,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=295",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.17.8.A, row 'Claude Opus 4.8', column 'Pass@1' = 59.9. Internal harness, 108 tasks, 3 trials, adaptive thinking at max effort; ~3 points above upstream-equivalent harness per card; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "toolathlon",
    "score": 0.593,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=295",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.17.8.A, row 'Claude Opus 4.7', column 'Pass@1' = 59.3. Internal harness, 108 tasks, 3 trials, adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "toolathlon",
    "score": 0.568,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=295",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.17.8.A, row 'Claude Opus 4.6', column 'Pass@1' = 56.8. Internal harness, 108 tasks, 3 trials, adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.813,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-Bench Pro', column 'Claude Sonnet 5.5' = 81.3 (also §8.2: 'Sonnet 5.5 achieved 81.3%'); standard configuration per caption: adaptive thinking at max effort, default sampling settings (temperature, top_p), averaged over 5 trials, context ≤1M; public/commercial split not stated (§8.2: 'draws problems from actively-maintained repositories with large, multi-file diffs'); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "swe-bench-multilingual",
    "score": 0.903,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-Bench Multilingual', column 'Claude Sonnet 5.5' = 90.3 (§8.2: 300 problems across 9 languages); adaptive thinking at max effort, default sampling, averaged over 5 trials; scaffold not named; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "hle",
    "score": 0.569,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Claude Sonnet 5.5' = 56.9; §8.11.1: full 2,500-question multimodal set, reasoning-only, thinking set to auto, 980k-token task budget per attempt, no context compaction, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "hle",
    "score": 0.431,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Claude Sonnet 5' = 43.1 (the Sonnet 5 System Card prints 43.2); full 2,500-question set, no tools, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "hle-with-tools",
    "score": 0.645,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Claude Sonnet 5.5' = 64.5; §8.11.1: full 2,500-question set with web search, web fetch, programmatic tool calling and code execution, thinking auto, 980k-token task budget, no compaction, HLE-source blocklist, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "hle-with-tools",
    "score": 0.549,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=109",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Claude Sonnet 5' = 54.9 (the Sonnet 5 System Card prints 57.4); full 2,500-question set with search/fetch/code tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "deepswe-v1-1",
    "score": 0.71,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=110",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, §8.3 'DeepSWE v1.1' prose: 'Sonnet 5.5 scored an average of 71.0% over five trials' (113 tasks); harness not named; standard configuration adaptive thinking at max effort per Table 8.1.A caption; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "terminal-bench-4",
    "score": 0.706,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=113",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, §8.5 'Terminal-Bench 4.0' prose: 'Claude Sonnet 5.5 scored 70.6% on Terminal-Bench 4.0'; 66 tasks; Claude Code in --bare mode at max thinking effort; 5 trials per task (330 trials), SE ±2.5 pts; safeguards enabled with default server-side fallback (1.2% of requests, 1.5% of trials); no internet egress (resources pre-cached); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "chartography-with-tools",
    "score": 0.902,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=125",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, §8.13.1 'Chartography' prose: 'Claude Sonnet 5.5 achieved a score of 61.6% without tools and 90.2% with tools'; with-tools figure recorded; adaptive thinking at max effort; tools = container with image file and standard libraries plus an image cropping tool; internal grading implementation, Gemini 3.5 Flash grader (as on the Surge AI leaderboard); averaged over five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "toolathlon-verified",
    "score": 0.778,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=135",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, Table 8.14.5.A 'Toolathlon scores (internal harness)', row 'Claude Sonnet 5.5', column 'Pass@1' = 77.8; Toolathlon-Verified (authors' final release, June 2026), 108 tasks; adaptive thinking at max effort with safety classifiers (2 of 324 trials stopped, counted as failures); Pass@1 averaged over three trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "healthbench",
    "score": 0.694,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=138",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, §8.15.1 'HealthBench' prose: 'Before length adjustment ... slightly ahead at max (69.4% against 68.1%)'; raw (not length-adjusted) score at max effort; safety classifiers enabled; graded by Claude Opus 4.8; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "global-mmlu",
    "score": 0.921,
    "source": "https://www-cdn.anthropic.com/870c8f525702625d2c62fc6dd04c857e3250bec1/Claude%20Sonnet%205.5%20System%20Card.pdf#page=140",
    "by": "lab",
    "date": "2026-09-28",
    "note": "Anthropic Claude Sonnet 5.5 System Card, §8.16.1 'GMMLU results' prose: 'average accuracy of 92.1% across 42 languages'; adaptive thinking at max effort, single trial, no tools or custom system prompt (Figure 8.16.1.A caption); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.632,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Claude Sonnet 5' = 63.2; standard configuration per caption: adaptive thinking at max effort, default sampling (temperature, top_p), averaged over 5 trials, 1M-token context; public/commercial split not stated (§8.2 'a harder variant ... reduced public ground-truth leakage'); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.581,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Claude Sonnet 4.6' = 58.1; Sonnet 4.6 settings not restated in this card; split not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "terminal-bench-2.1",
    "score": 0.804,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Terminal-Bench 2.1', column 'Claude Sonnet 5' = 80.4; §8.3: mini-SWE-agent harness (not Terminus-2) on a GKE cluster with 1× timeout and 3× memory ceiling, xhigh effort, mean reward averaged over 5 attempts on each of 89 tasks (445 trials); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "terminal-bench-2.1",
    "score": 0.67,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Terminal-Bench 2.1', column 'Claude Sonnet 4.6' = 67.0; §8.3: 'Sonnet 4.6 achieved a 67% score on the same evaluation and same infrastructure, at high effort' (mini-SWE-agent, GKE); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "browsecomp-context-managed",
    "score": 0.847,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'BrowseComp', column 'Claude Sonnet 5' = 84.7 (single agent); caption and §8.9.2: web search, web fetch, programmatic tool calling and code execution, adaptive thinking at max effort, 10M-token limit with context compaction triggered at 200k, evaluation blocklist; multi-agent 86.6 not recorded; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "hle",
    "score": 0.432,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Claude Sonnet 5' = 43.2 (the later Sonnet 5.5 card prints 43.1); §8.9.1: full 2,500-question multimodal set, reasoning only, thinking auto, 1M total-token cap, no compaction, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "hle",
    "score": 0.346,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Claude Sonnet 4.6' = 34.6 (re-graded with updated grader per the Sonnet 5 announcement); no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "hle-with-tools",
    "score": 0.574,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Claude Sonnet 5' = 57.4 (the later Sonnet 5.5 card prints 54.9); §8.9.1: web search, web fetch, programmatic tool calling, code execution, thinking auto, 1M total-token cap, no compaction, HLE-source blocklist, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "hle-with-tools",
    "score": 0.468,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Claude Sonnet 4.6' = 46.8; with search/fetch/code tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "osworld-verified",
    "score": 0.812,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'OSWorld-Verified', column 'Claude Sonnet 5' = 81.2; §8.10.2 / Fig. 8.10.2.A: 361 tasks, 1080p, max 100 action steps, adaptive thinking at max effort, first-attempt success (pass@1) averaged over five runs; zoom-tool batched-action bug fix and 128K per-turn token limit (footnote 14); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "osworld-verified",
    "score": 0.785,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'OSWorld-Verified', column 'Claude Sonnet 4.6' = 78.5; re-evaluated with the zoom-tool fix and 128K per-turn token limit (footnote 14, §8.10.2); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "swe-bench-verified",
    "score": 0.852,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=116",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.2 prose: 'SWE-bench Verified is a 500-problem subset ... Claude Sonnet 5 achieved 85.2%'; standard configuration (adaptive thinking at max effort), average over five trials, thinking blocks included; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "swe-bench-multilingual",
    "score": 0.783,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=116",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.2 prose: 'SWE-bench Multilingual extends the format to 300 problems across 9 programming languages. Sonnet 5 achieved 78.3%'; standard configuration (adaptive thinking at max effort), average over five trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "usamo-2026",
    "score": 0.795,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=120",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.6 'USAMO 2026' prose: 'Claude Sonnet 5 scored 79.5%, averaging over 10 attempts per problem'; high effort with a 300k-token limit; MathArena-style grading: proofs rewritten by Gemini 3.1 Pro and judged by a panel (Gemini 3.1 Pro, Claude Opus 4.6, Claude Mythos Preview), final score = minimum judge score; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "usamo-2026",
    "score": 0.55,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=120",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.6 'USAMO 2026' prose: 'Under similar settings, Sonnet 4.6 scored 55.0%'; same panel grading (minimum judge score); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "charxiv-rq",
    "score": 0.77,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=129",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.10.5 'CharXiv Reasoning' prose: 'Claude Sonnet 5 achieved 77.0% without tools and 88.3% with tools'; no-tools figure recorded; 1,000 validation-split questions, reference prompts, Claude Sonnet 4.6 grader instead of GPT-4o, adaptive thinking at max effort, averaged over five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "charxiv-rq",
    "score": 0.716,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=129",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, §8.10.5 'CharXiv Reasoning' prose: 'Claude Sonnet 4.6 (71.6% and 85.3%)' (without/with tools); no-tools figure recorded; 1,000 validation questions, Claude Sonnet 4.6 grader, adaptive thinking max effort, five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "toolathlon",
    "score": 0.543,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=135",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.11.5.A 'Toolathlon scores using an internal harness', row 'Claude Sonnet 5', column 'Pass@1' = 54.3; original Toolathlon release (108 tasks; card notes roughly a quarter of tasks unsatisfiable as published), not Toolathlon-Verified; adaptive thinking at max effort; Pass@1 averaged over 3 trials; pinned data feeds/containers, scores run ~3 points above a strictly upstream-equivalent harness; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "toolathlon",
    "score": 0.494,
    "source": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf#page=135",
    "by": "lab",
    "date": "2026-06-30",
    "note": "Anthropic Claude Sonnet 5 System Card, Table 8.11.5.A 'Toolathlon scores using an internal harness', row 'Claude Sonnet 4.6', column 'Pass@1' = 49.4; original Toolathlon (not Verified); adaptive thinking max effort; Pass@1 over 3 trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.792,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Pro', column 'Opus 5' = 79.2 (also §8.2); standard configuration per caption: adaptive thinking at max effort, default sampling (temperature, top_p), averaged over 5 trials, context ≤1M; split not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "swe-bench-multilingual",
    "score": 0.895,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Multilingual', column 'Opus 5' = 89.5 (also §8.2); adaptive thinking at max effort, averaged over 5 trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "swe-bench-multilingual",
    "score": 0.844,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Multilingual', column 'Opus 4.8' = 84.4; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "swe-bench-multilingual",
    "score": 0.866,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-bench Multilingual', column 'Fable 5' = 86.6; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "deepswe-v1-1",
    "score": 0.688,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'DeepSWE v1.1', column 'Opus 5' = 68.8 (§8.3: 'an average of 68.8% over five trials', 113 tasks); harness not named; adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "deepswe-v1-1",
    "score": 0.59,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'DeepSWE v1.1', column 'Opus 4.8' = 59.0; harness not named; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "deepswe-v1-1",
    "score": 0.697,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'DeepSWE v1.1', column 'Fable 5' = 69.7; harness not named; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "browsecomp-context-managed",
    "score": 0.908,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'BrowseComp', column 'Opus 5' = 90.8; §8.10.2: web search, web fetch, programmatic tool calling and code execution, context compaction triggered at 200k tokens to extend beyond the 1M window (token budgets up to 10M in Fig. 8.10.2.A), Claude Opus 4.7 grader; Fig. 8.10.2.A says Opus 5 used an unreleased effort configuration; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "browsecomp-context-managed",
    "score": 0.843,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'BrowseComp', column 'Opus 4.8' = 84.3; Fig. 8.10.2.A: comparison models run at max effort with token budgets scaled 1M→10M (context compaction); detailed settings deferred to the Fable 5 System Card; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "browsecomp-context-managed",
    "score": 0.874,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'BrowseComp', column 'Fable 5' = 87.4; Fig. 8.10.2.A: comparison models run at max effort with token budgets scaled 1M→10M (context compaction); detailed settings deferred to the Fable 5 System Card; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "hle",
    "score": 0.563,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Opus 5' = 56.3 (later Anthropic pages print 56.6 for Opus 5); §8.10.1: full 2,500-question multimodal set, reasoning only, thinking auto, 1M total-token cap, no compaction, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "hle",
    "score": 0.565,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / No tools', column 'Fable 5' = 56.5; no tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "hle-with-tools",
    "score": 0.647,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Opus 5' = 64.7 (later Anthropic pages print 63.6 for Opus 5); §8.10.1: web search, web fetch, programmatic tool calling and code execution, thinking auto, 1M total-token cap, no compaction, HLE-source blocklist, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-fable-5",
    "bench": "hle-with-tools",
    "score": 0.639,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=152",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.1.A 'Capability evaluation summary', row 'Humanity’s Last Exam / With tools', column 'Fable 5' = 63.9; with search/fetch/code tools; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "swe-bench-verified",
    "score": 0.96,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=153",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.2 prose: 'SWE-bench Verified is a 500-problem subset ... Claude Opus 5 achieved 96.0%'; average over five trials, standard configuration (adaptive thinking at max effort); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "chartography-with-tools",
    "score": 0.83,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.12.1 'Chartography' prose: 'Claude Opus 5 achieved a score of 29.6% without tools and 83.0% with tools'; with-tools figure recorded; adaptive thinking at max effort, container with image file and standard libraries plus image cropping tool, internal grading implementation, averaged over five runs; grader not named here (the Opus 5.5 card says earlier cards used a Claude Sonnet 4.6 judge for Claude models and re-reports Opus 5 at 83.4%); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "chartography-with-tools",
    "score": 0.75,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=174",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.12.1 'Chartography' prose: 'Claude Opus 4.8 scored 17.0% and 75.0%' (without/with tools); with-tools figure recorded; adaptive thinking max effort, container + image cropping tool, five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "toolathlon-verified",
    "score": 0.806,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=183",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.13.6.A 'Toolathlon scores (internal harness)', row 'Claude Opus 5', column 'Pass@1' = 80.6; Toolathlon-Verified (authors' final release, June 2026), 108 tasks; internal harness mirroring task definitions/checkers, adaptive thinking at max effort, no safety classifiers or fallback; Pass@1 averaged over three trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "toolathlon-verified",
    "score": 0.799,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=183",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.13.6.A 'Toolathlon scores (internal harness)', row 'Claude Opus 4.8', column 'Pass@1' = 79.9; Toolathlon-Verified, internal harness, adaptive thinking max effort; Pass@1 over three trials (published leaderboard 76.2%, null attempts counted as failures); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "toolathlon-verified",
    "score": 0.747,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=183",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, Table 8.13.6.A 'Toolathlon scores (internal harness)', row 'Claude Sonnet 5', column 'Pass@1' = 74.7; Toolathlon-Verified, internal harness, adaptive thinking max effort; Pass@1 over three trials (published leaderboard 71.6%); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-5",
    "bench": "healthbench",
    "score": 0.671,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.15.1 'HealthBench results' prose: 'Claude Opus 5 achieved a raw score of 67.1%' (length-adjusted 57.8% not recorded); full set of 5,000 conversations; adaptive thinking at max effort, Claude Opus 4.8 grader, averaged over 5 trials, no tools or custom system prompt; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "healthbench",
    "score": 0.588,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.15.1 'HealthBench results' prose: 'Claude Opus 4.8 at 58.8%' (raw score); adaptive thinking max effort, Claude Opus 4.8 grader, 5 trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "healthbench",
    "score": 0.592,
    "source": "https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Anthropic Claude Opus 5 System Card, §8.15.1 'HealthBench results' prose: 'Claude Sonnet 5 at 59.2%' (raw score); adaptive thinking max effort, Claude Opus 4.8 grader, 5 trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.908,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'Terminal-bench 2.1', column '3.8 Flash' = 90.8%; harness, thinking level and trials not stated on this page (the 3.8 Flash model card gives 89.4% for the same label, self-computed with Terminus 2); page 'Last updated 2026-10-06 UTC'; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.816,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'Terminal-bench 2.1', column '3.7 Flash' = 81.6%; settings not stated. Conflicts with the 3.7 Flash guide (85.1%) and both model cards (85.8%) for the same model and label; page 'Last updated 2026-10-06 UTC'; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.616,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'SWE-Bench Pro', column '3.8 Flash' = 61.6%; public/commercial split, harness and thinking level not stated; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.604,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'SWE-Bench Pro', column '3.7 Flash' = 60.4%; split, harness and thinking level not stated; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "tau3-banking",
    "score": 0.381,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'τ³-bench Banking', column '3.8 Flash' = 38.1%; user simulator, harness and thinking level not stated; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "tau3-banking",
    "score": 0.309,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-8-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.8 Flash', table 'What's new in Gemini 3.8 Flash?', row 'τ³-bench Banking', column '3.7 Flash' = 30.9%; user simulator, harness and thinking level not stated; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.851,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-7-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.7 Flash', table 'What's new in Gemini 3.7 Flash?', row 'Terminal-bench 2.1', column '3.7 Flash' = 85.1%; settings not stated. Conflicts with the 3.8 Flash guide (81.6%) and both model cards (85.8%); this is the model's own guide; page 'Last updated 2026-10-06 UTC'; CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.73,
    "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-3-7-flash#whats-new",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Google Cloud 'Developer's guide to Gemini 3.7 Flash', table 'What's new in Gemini 3.7 Flash?', row 'Terminal-bench 2.1', column '3.6 Flash' = 73.0%; settings not stated. Conflicts with the 3.7 Flash model card (78.0%, self-computed, Terminus 2); CC BY 4.0 page (rule a).",
    "source_id": "google-cloud-gemini-guides"
  },
  {
    "model": "gemini-4-argon",
    "bench": "deepswe-v1-1",
    "score": 0.779,
    "source": "https://deepmind.google/models/gemini/#performance",
    "by": "lab",
    "note": "Google DeepMind Gemini model page, 'Frontier performance in complex workflows' table, row 'DeepSWE v1.1 (Agentic coding)', column 'Gemini 4 Argon' = 77.9%; per the Gemini 4 Argon evaluation PDF (storage.googleapis.com/deepmind-media/gemini/gemini_4_argon_model_evaluation.pdf, p.2) self-computed with a mini-swe agent harness, Gemini API at highest thinking setting, pass@1, averaged over trials for smaller benchmarks; same figure in the 2026-09-30 blog.google Argon post text. Page undated (Argon announced 2026-09-30); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-4-argon",
    "bench": "terminal-bench-4",
    "score": 0.574,
    "source": "https://deepmind.google/models/gemini/#performance",
    "by": "lab",
    "note": "Google DeepMind Gemini model page, 'Frontier performance in complex workflows' table, row 'Terminal-bench 4.0 (Agentic coding)', column 'Gemini 4 Argon' = 57.4%; per the Argon evaluation PDF (p.2) the Argon result is self-computed (other models from the public leaderboard), Gemini API at highest thinking setting, pass@1; harness not stated. Page undated; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-4-argon",
    "bench": "lvbench",
    "score": 0.917,
    "source": "https://deepmind.google/models/gemini/#performance",
    "by": "lab",
    "note": "Google DeepMind Gemini model page, 'Frontier performance in complex workflows' table, row 'LVBench (Multimodal understanding)', column 'Gemini 4 Argon' = 91.7%; per the Argon evaluation PDF (p.4) self-computed without tools at 1 FPS, highest thinking setting, pass@1; same figure in the blog.google Argon post. Page undated; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "deepswe-v1-1",
    "score": 0.737,
    "source": "https://deepmind.google/models/model-cards/gemini-3-8-flash/#evaluation",
    "by": "lab",
    "date": "2026-09-02",
    "note": "Gemini 3.8 Flash model card (HTML), Evaluation 'Results as of September, 2026' table, row 'DeepSWE v1.1 (Long-horizon software engineering)', column 'Gemini 3.8 Flash' = 73.7%; per the 3.8 Flash evaluation PDF (storage.googleapis.com/deepmind-media/gemini/gemini_3-8_flash_model_evaluation.pdf, p.2) self-computed with a mini-swe agent harness and high thinking, pass@1, default sampling. The PDF model card shows this table only as an image; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.894,
    "source": "https://deepmind.google/models/model-cards/gemini-3-8-flash/#evaluation",
    "by": "lab",
    "date": "2026-09-02",
    "note": "Gemini 3.8 Flash model card (HTML), Evaluation results table, row 'Terminal-bench 2.1 (Agentic terminal coding)', column 'Gemini 3.8 Flash' = 89.4%; per the 3.8 Flash evaluation PDF (p.2) self-computed for Gemini models with the default Terminus 2 harness, pass@1, default sampling. Differs from the Cloud guide's 90.8% for the same label; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "charxiv-rq",
    "score": 0.862,
    "source": "https://deepmind.google/models/model-cards/gemini-3-8-flash/#evaluation",
    "by": "lab",
    "date": "2026-09-02",
    "note": "Gemini 3.8 Flash model card (HTML), Evaluation results table, row 'CharXiv Reasoning (Information synthesis from complex charts)', notes 'No tools', column 'Gemini 3.8 Flash' = 86.2%; per the evaluation PDF (p.2) self-computed, pass@1, default sampling; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "lvbench",
    "score": 0.871,
    "source": "https://deepmind.google/models/model-cards/gemini-3-8-flash/#evaluation",
    "by": "lab",
    "date": "2026-09-02",
    "note": "Gemini 3.8 Flash model card (HTML), Evaluation results table, row 'LVBench (Long video understanding)', column 'Gemini 3.8 Flash', second value '87.1%(static)' (the cell also shows 87.8% (agentic), listed as unmatched); per the evaluation PDF (p.2) self-computed without tools, 1024 frames for Gemini; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "deepswe-v1-1",
    "score": 0.653,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation 'Results as of August 2026' table, row 'DeepSWE v1.1 (Long-horizon software engineering)', column 'Gemini 3.7 Flash' = 65.3%; per the 3.7 Flash evaluation PDF (storage.googleapis.com/deepmind-media/gemini/gemini_3-7_flash_model_evaluation.pdf, p.2) self-computed with a mini swe agent harness, LiteLLM 1.96, high thinking, pass@1. The PDF model card shows this table only as an image; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.858,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'Terminal-bench 2.1 (Agentic terminal coding)', column 'Gemini 3.7 Flash' = 85.8%; per the evaluation PDF (p.2) self-computed for Gemini models with the default Terminus 2 harness, pass@1, default sampling. Same value in the 3.8 Flash card; Cloud guides print 85.1% (3.7 guide) and 81.6% (3.8 guide); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.78,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'Terminal-bench 2.1', column 'Gemini 3.6 Flash' = 78.0%; self-computed for Gemini models with the default Terminus 2 harness (evaluation PDF p.2). The 3.7 Cloud guide prints 73.0% for 3.6 Flash; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "terminal-bench-3",
    "score": 0.149,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'Terminal-bench 3.0 (General agent capabilities)', column 'Gemini 3.7 Flash' = 14.9%; per the evaluation PDF (p.2) Gemini results self-computed with a mini swe agent harness (LiteLLM 1.96), with minor compatibility modifications to the cli-2ph-simplex and vpp-loss-divergence tasks; pass@1; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "terminal-bench-3",
    "score": 0.054,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'Terminal-bench 3.0', column 'Gemini 3.6 Flash' = 5.4%; Gemini results self-computed with a mini swe agent harness (LiteLLM 1.96), minor modifications to two tasks (evaluation PDF p.2); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "charxiv-rq",
    "score": 0.845,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'CharXiv Reasoning (Information synthesis from complex charts)', notes 'No tools', column 'Gemini 3.7 Flash' = 84.5%; self-computed (evaluation PDF p.2); same value in the 3.8 Flash card; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "charxiv-rq",
    "score": 0.852,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'CharXiv Reasoning', notes 'No tools', column 'Gemini 3.6 Flash' = 85.2%; self-computed (evaluation PDF p.2); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "lvbench",
    "score": 0.854,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'LVBench (Long video understanding)', column 'Gemini 3.7 Flash' = 85.4%; per the evaluation PDF (p.3) self-computed, 1024 frames for Gemini, no tool used; same value in the 3.8 Flash card; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "lvbench",
    "score": 0.842,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'LVBench', column 'Gemini 3.6 Flash' = 84.2%; self-computed, 1024 frames, no tool used (evaluation PDF p.3); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.97,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'GDM-MRCR v2 (8-needle) (Long context performance)', notes '128k (average)', column 'Gemini 3.7 Flash' = 97.0%; per the evaluation PDF (p.3) self-computed, 128k reported as a cumulative score. Mapped to mrcr-v2-8needle-128k assuming the catalogue's Gemma-report entry uses the same Google 128k average (flagged); vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.918,
    "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/#evaluation",
    "by": "lab",
    "date": "2026-08-13",
    "note": "Gemini 3.7 Flash model card (HTML), Evaluation results table, row 'GDM-MRCR v2 (8-needle)', notes '128k (average)', column 'Gemini 3.6 Flash' = 91.8%; self-computed, cumulative 128k score (evaluation PDF p.3); same mapping caveat as the 3.7 Flash row; vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "agents-last-exam-unspecified",
    "score": 0.504,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Professional: Agents' Last Exam' / column 'GPT-5.6 Terra'; value printed as '50.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The page states neither the metric nor the harness, hence the -unspecified catalogue entry.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "agents-last-exam-unspecified",
    "score": 0.503,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Professional: Agents' Last Exam' / column 'GPT-5.6 Luna'; value printed as '50.3%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The page states neither the metric nor the harness, hence the -unspecified catalogue entry.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "agents-last-exam-unspecified",
    "score": 0.469,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Professional: Agents' Last Exam' / column 'GPT-5.5'; value printed as '46.9%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The page states neither the metric nor the harness, hence the -unspecified catalogue entry.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.646,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro' / column 'GPT-5.6 Sol'; value printed as '64.6%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The table does not say Public or Commercial; OpenAI's GPT-5.5 post labels its own GPT-5.5 figure 'SWE-Bench Pro (Public)' 58.6%, a different run.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.634,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro' / column 'GPT-5.6 Terra'; value printed as '63.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The table does not say Public or Commercial; OpenAI's GPT-5.5 post labels its own GPT-5.5 figure 'SWE-Bench Pro (Public)' 58.6%, a different run.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.627,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro' / column 'GPT-5.6 Luna'; value printed as '62.7%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The table does not say Public or Commercial; OpenAI's GPT-5.5 post labels its own GPT-5.5 figure 'SWE-Bench Pro (Public)' 58.6%, a different run.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.594,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro' / column 'GPT-5.5'; value printed as '59.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The table does not say Public or Commercial; OpenAI's GPT-5.5 post labels its own GPT-5.5 figure 'SWE-Bench Pro (Public)' 58.6%, a different run.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "deepswe-v1-1",
    "score": 0.727,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: DeepSWE v1.1' / column 'GPT-5.6 Sol'; value printed as '72.7%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "deepswe-v1-1",
    "score": 0.696,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: DeepSWE v1.1' / column 'GPT-5.6 Terra'; value printed as '69.6%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "deepswe-v1-1",
    "score": 0.672,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: DeepSWE v1.1' / column 'GPT-5.6 Luna'; value printed as '67.2%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "deepswe-v1-1",
    "score": 0.67,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: DeepSWE v1.1' / column 'GPT-5.5'; value printed as '67%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "terminal-bench-2.1",
    "score": 0.888,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.1' / column 'GPT-5.6 Sol'; value printed as '88.8%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated; single-agent columns only.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "terminal-bench-2.1",
    "score": 0.874,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.1' / column 'GPT-5.6 Terra'; value printed as '87.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated; single-agent columns only.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "terminal-bench-2.1",
    "score": 0.847,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.1' / column 'GPT-5.6 Luna'; value printed as '84.7%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated; single-agent columns only.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "terminal-bench-2.1",
    "score": 0.856,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.1' / column 'GPT-5.5'; value printed as '85.6%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated; single-agent columns only.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "browsecomp",
    "score": 0.904,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use: BrowseComp' / column 'GPT-5.6 Sol'; value printed as '90.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "browsecomp",
    "score": 0.875,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use: BrowseComp' / column 'GPT-5.6 Terra'; value printed as '87.5%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "browsecomp",
    "score": 0.833,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use: BrowseComp' / column 'GPT-5.6 Luna'; value printed as '83.3%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "frontiermath",
    "score": 0.89,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 1-3 (v2)' / column 'GPT-5.6 Sol'; value printed as '89%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.8526) and GPT-5.6 Sol's 89% matches Epoch's 0.8912 after rounding; the page does not credit Epoch, so these may be the same runs.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "frontiermath",
    "score": 0.849,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 1-3 (v2)' / column 'GPT-5.6 Terra'; value printed as '84.9%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.8526) and GPT-5.6 Sol's 89% matches Epoch's 0.8912 after rounding; the page does not credit Epoch, so these may be the same runs.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "frontiermath",
    "score": 0.786,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 1-3 (v2)' / column 'GPT-5.6 Luna'; value printed as '78.6%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.8526) and GPT-5.6 Sol's 89% matches Epoch's 0.8912 after rounding; the page does not credit Epoch, so these may be the same runs.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.83,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 4 (v2)' / column 'GPT-5.6 Sol'; value printed as '83%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.725); the page does not credit Epoch.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.683,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 4 (v2)' / column 'GPT-5.6 Terra'; value printed as '68.3%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.725); the page does not credit Epoch.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.585,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: FrontierMath Tier 4 (v2)' / column 'GPT-5.6 Luna'; value printed as '58.5%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Tool use not stated. Caution: the GPT-5.5 cell equals Epoch AI's published value already in data (0.725); the page does not credit Epoch.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "toolathlon",
    "score": 0.58,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.6 Sol'; value printed as '58%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric not stated (Toolathlon's standard metric is pass@1).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "toolathlon",
    "score": 0.531,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.6 Terra'; value printed as '53.1%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric not stated (Toolathlon's standard metric is pass@1).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "toolathlon",
    "score": 0.534,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.6 Luna'; value printed as '53.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric not stated (Toolathlon's standard metric is pass@1).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "mmmu-pro",
    "score": 0.83,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Multimodal: MMMU Pro (no tools)' / column 'GPT-5.6 Sol'; value printed as '83%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. MMMU-Pro subset/aggregation not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "mmmu-pro",
    "score": 0.807,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Multimodal: MMMU Pro (no tools)' / column 'GPT-5.6 Terra'; value printed as '80.7%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. MMMU-Pro subset/aggregation not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "mmmu-pro",
    "score": 0.784,
    "source": "https://openai.com/index/gpt-5-6",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6: Frontier intelligence that scales with your ambition' (July 9, 2026), evaluation tables at the end of the post; read from Internet Archive capture 20261003080126 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Multimodal: MMMU Pro (no tools)' / column 'GPT-5.6 Luna'; value printed as '78.4%'. Settings: reasoning effort not stated for the table (the post describes max and xhigh settings); the separate 'GPT-5.6 Sol Ultra' four-agent column is not used. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. MMMU-Pro subset/aggregation not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "agents-last-exam-unspecified",
    "score": 0.593,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer Use: Agents' Last Exam' / column 'GPT-6 Astra'; value printed as '59.3%'. Settings: 'maximum at any effort' per the page note. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric and harness not stated, hence the -unspecified catalogue entry.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "screenspot-pro",
    "score": 0.927,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer Use: ScreenSpot-Pro (no tools)' / column 'GPT-6 Astra'; value printed as '92.7%'. Settings: 'maximum at any effort' per the page note; no tools. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "screenspot-pro",
    "score": 0.769,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer Use: ScreenSpot-Pro (no tools)' / column 'GPT-5.6 Sol'; value printed as '76.9%'. Settings: 'maximum at any effort' per the page note; no tools. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Footnote 2: 'GPT-5.6 Sol refers to the version available in our API, ChatGPT Codex, and ChatGPT Work.'",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "browsecomp",
    "score": 0.915,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Professional: BrowseComp' / column 'GPT-6 Astra'; value printed as '91.5%'. Settings: 'maximum at any effort' per the page note. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "terminal-bench-4",
    "score": 0.579,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 4.0' / column 'GPT-6 Astra'; value printed as '57.9%'. Settings: 'maximum at any effort' per the page note; harness not stated. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "deepswe-v1-1",
    "score": 0.741,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: DeepSWE v1.1' / column 'GPT-6 Astra'; value printed as '74.1%'. Settings: 'maximum at any effort' per the page note; harness not stated. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "hle-with-tools",
    "score": 0.572,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026; date from OpenAI's post listing and the system card), evaluation tables. Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; read from Internet Archive capture 20261004191257 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (w/ tools)' / column 'GPT-6 Astra'; value printed as '57.2%'. Settings: 'maximum at any effort' per the page note; with tools; tool set and subset (full vs text-only) not stated. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-sol",
    "bench": "agents-last-exam-unspecified",
    "score": 0.564,
    "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'Introducing GPT-6 Sol and Luna' (Sep 22, 2026; date from OpenAI's post listing); read from Internet Archive capture 20261003152252 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Professional work text: 'On Agents' Last Exam ... GPT-6 Sol at max effort scores 56.4%''; value printed as '56.4%'. Settings: max effort. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Footnote names 'Agents' Last Exam V1'; metric and harness not stated, hence the -unspecified catalogue entry.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-sol",
    "bench": "deepswe-v1-1",
    "score": 0.688,
    "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'Introducing GPT-6 Sol and Luna' (Sep 22, 2026; date from OpenAI's post listing); read from Internet Archive capture 20261003152252 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding text: 'On DeepSWE v1.1 ... GPT-6 Sol at max effort scores 68.8%''; value printed as '68.8%'. Settings: max effort; harness not stated. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-luna",
    "bench": "deepswe-v1-1",
    "score": 0.666,
    "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'Introducing GPT-6 Sol and Luna' (Sep 22, 2026; date from OpenAI's post listing); read from Internet Archive capture 20261003152252 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding text: 'GPT-6 Luna at max effort scores 66.6%' (DeepSWE v1.1)'; value printed as '66.6%'. Settings: max effort; harness not stated. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "swe-bench-pro",
    "score": 0.586,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro (Public) *' / column 'GPT-5.5'; value printed as '58.6%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page footnote: '*Labs have noted evidence of memorization on this eval'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "terminal-bench-2",
    "score": 0.827,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.0' / column 'GPT-5.5'; value printed as '82.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "osworld-verified",
    "score": 0.787,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use and vision: OSWorld-Verified' / column 'GPT-5.5'; value printed as '78.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "mmmu-pro",
    "score": 0.812,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use and vision: MMMU Pro (no tools)' / column 'GPT-5.5'; value printed as '81.2%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Same value appears for GPT-5.5 on the GPT-5.6 post; recorded once from GPT-5.5's own post.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "browsecomp",
    "score": 0.844,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: BrowseComp' / column 'GPT-5.5'; value printed as '84.4%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5-pro",
    "bench": "browsecomp",
    "score": 0.901,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: BrowseComp' / column 'GPT-5.5 Pro'; value printed as '90.1%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "toolathlon",
    "score": 0.556,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.5'; value printed as '55.6%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "tau2-telecom",
    "score": 0.98,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Tau2-bench Telecom*** (original prompts)' / column 'GPT-5.5'; value printed as '98.0%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page: 'Tau2-bench Telecom was run without prompt tuning (and GPT-4.1 as user model)'; footnote '***': 'results for 5.5 and 5.4 with original prompts i.e no prompt adjustment'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "tau2-telecom",
    "score": 0.928,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Tau2-bench Telecom*** (original prompts)' / column 'GPT-5.4'; value printed as '92.8%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page: 'Tau2-bench Telecom was run without prompt tuning (and GPT-4.1 as user model)'; footnote '***': 'results for 5.5 and 5.4 with original prompts i.e no prompt adjustment'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "hle",
    "score": 0.414,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (no tools)' / column 'GPT-5.5'; value printed as '41.4%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset (full vs text-only) not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5-pro",
    "bench": "hle",
    "score": 0.431,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (no tools)' / column 'GPT-5.5 Pro'; value printed as '43.1%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset (full vs text-only) not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "hle-with-tools",
    "score": 0.522,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (with tools)' / column 'GPT-5.5'; value printed as '52.2%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5-pro",
    "bench": "hle-with-tools",
    "score": 0.572,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (with tools)' / column 'GPT-5.5 Pro'; value printed as '57.2%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "cybergym",
    "score": 0.818,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Cybersecurity: CyberGym' / column 'GPT-5.5'; value printed as '81.8%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric (success rate) not restated; harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "cybergym",
    "score": 0.79,
    "source": "https://openai.com/index/introducing-gpt-5-5",
    "by": "lab",
    "date": "2026-04-23",
    "note": "OpenAI 'Introducing GPT-5.5' (April 23, 2026), Evaluations tables. Page note: 'Evals of GPT were run with reasoning effort set to xhigh and were conducted in a research environment'; read from Internet Archive capture 20260930123856 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Cybersecurity: CyberGym' / column 'GPT-5.4'; value printed as '79.0%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Metric (success rate) not restated; harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "swe-bench-pro",
    "score": 0.577,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-Bench Pro (Public)' / column 'GPT-5.4'; value printed as '57.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "terminal-bench-2",
    "score": 0.751,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.0' / column 'GPT-5.4'; value printed as '75.1%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "osworld-verified",
    "score": 0.75,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use and vision: OSWorld-Verified' / column 'GPT-5.4'; value printed as '75.0%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "mmmu-pro",
    "score": 0.812,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Computer use and vision: MMMU Pro (no tools)' / column 'GPT-5.4'; value printed as '81.2%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Text: 'MMMUPro was run with reasoning effort set to xhigh'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "browsecomp",
    "score": 0.827,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: BrowseComp' / column 'GPT-5.4'; value printed as '82.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page: search blocklist excluding answer-bearing sites; 'Models use the ChatGPT search tool, which can have small differences from API search'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "browsecomp",
    "score": 0.893,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: BrowseComp' / column 'GPT-5.4 Pro'; value printed as '89.3%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page: search blocklist excluding answer-bearing sites; 'Models use the ChatGPT search tool, which can have small differences from API search'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "browsecomp",
    "score": 0.773,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: BrowseComp' / column 'GPT-5.3-Codex'; value printed as '77.3%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Page: search blocklist excluding answer-bearing sites; 'Models use the ChatGPT search tool, which can have small differences from API search'.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "toolathlon",
    "score": 0.546,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.4'; value printed as '54.6%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "toolathlon",
    "score": 0.519,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool use: Toolathlon' / column 'GPT-5.3-Codex'; value printed as '51.9%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "frontierscience-research",
    "score": 0.33,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Frontier Science Research' / column 'GPT-5.4'; value printed as '33.0%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "frontierscience-research",
    "score": 0.367,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Frontier Science Research' / column 'GPT-5.4 Pro'; value printed as '36.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "hle",
    "score": 0.398,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (no tools)' / column 'GPT-5.4'; value printed as '39.8%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "hle",
    "score": 0.427,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (no tools)' / column 'GPT-5.4 Pro'; value printed as '42.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "hle-with-tools",
    "score": 0.521,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (with tools)' / column 'GPT-5.4'; value printed as '52.1%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "hle-with-tools",
    "score": 0.587,
    "source": "https://openai.com/index/introducing-gpt-5-4",
    "by": "lab",
    "date": "2026-03-05",
    "note": "OpenAI 'Introducing GPT-5.4' (March 5, 2026), Evaluations tables. Page note: 'Evals were run with reasoning effort set to xhigh, except where specified otherwise'; read from Internet Archive capture 20260923100550 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Academic: Humanity's Last Exam (with tools)' / column 'GPT-5.4 Pro'; value printed as '58.7%'. Settings: reasoning effort xhigh (page note). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "swe-bench-pro",
    "score": 0.544,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-bench Pro (Public)' / column 'GPT-5.4 mini (xhigh)'; value printed as '54.4%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "swe-bench-pro",
    "score": 0.524,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-bench Pro (Public)' / column 'GPT-5.4 nano (xhigh)'; value printed as '52.4%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "swe-bench-pro",
    "score": 0.457,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: SWE-bench Pro (Public)' / column 'GPT-5 mini (high)'; value printed as '45.7%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "terminal-bench-2",
    "score": 0.6,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.0' / column 'GPT-5.4 mini (xhigh)'; value printed as '60.0%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "terminal-bench-2",
    "score": 0.463,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.0' / column 'GPT-5.4 nano (xhigh)'; value printed as '46.3%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "terminal-bench-2",
    "score": 0.382,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Coding: Terminal-Bench 2.0' / column 'GPT-5 mini (high)'; value printed as '38.2%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "toolathlon",
    "score": 0.429,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool-calling: Toolathlon' / column 'GPT-5.4 mini (xhigh)'; value printed as '42.9%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "toolathlon",
    "score": 0.355,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool-calling: Toolathlon' / column 'GPT-5.4 nano (xhigh)'; value printed as '35.5%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "toolathlon",
    "score": 0.269,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Tool-calling: Toolathlon' / column 'GPT-5 mini (high)'; value printed as '26.9%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "hle-with-tools",
    "score": 0.415,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/ tool' / column 'GPT-5.4 mini (xhigh)'; value printed as '41.5%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "hle-with-tools",
    "score": 0.377,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/ tool' / column 'GPT-5.4 nano (xhigh)'; value printed as '37.7%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "hle-with-tools",
    "score": 0.316,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/ tool' / column 'GPT-5 mini (high)'; value printed as '31.6%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. With tools; tool set and subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "hle",
    "score": 0.282,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/o tools' / column 'GPT-5.4 mini (xhigh)'; value printed as '28.2%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "hle",
    "score": 0.243,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/o tools' / column 'GPT-5.4 nano (xhigh)'; value printed as '24.3%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "hle",
    "score": 0.183,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Intelligence: HLE w/o tools' / column 'GPT-5 mini (high)'; value printed as '18.3%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. No tools; subset not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "osworld-verified",
    "score": 0.721,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: OSWorld-Verified' / column 'GPT-5.4 mini (xhigh)'; value printed as '72.1%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "osworld-verified",
    "score": 0.39,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: OSWorld-Verified' / column 'GPT-5.4 nano (xhigh)'; value printed as '39.0%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "osworld-verified",
    "score": 0.42,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: OSWorld-Verified' / column 'GPT-5 mini (high)'; value printed as '42.0%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "mmmu-pro",
    "score": 0.766,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: MMMUPro' / column 'GPT-5.4 mini (xhigh)'; value printed as '76.6%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Plain 'MMMUPro' row (a separate 'MMMUPro w/ Python' row is not used).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "mmmu-pro",
    "score": 0.661,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: MMMUPro' / column 'GPT-5.4 nano (xhigh)'; value printed as '66.1%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Plain 'MMMUPro' row (a separate 'MMMUPro w/ Python' row is not used).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "mmmu-pro",
    "score": 0.675,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano' (March 17, 2026), benchmark tables with columns 'GPT-5.4 (xhigh)', 'GPT-5.4 mini (xhigh)', 'GPT-5.4 nano (xhigh)', 'GPT-5 mini (high)' (footnote: 'The highest reasoning_effort available for GPT-5 mini is high'); read from Internet Archive capture 20260929161725 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'MM / Vision / CUA: MMMUPro' / column 'GPT-5 mini (high)'; value printed as '67.5%'. Settings: reasoning effort high (the highest available for GPT-5 mini, per footnote 1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Plain 'MMMUPro' row (a separate 'MMMUPro w/ Python' row is not used).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "swe-bench-pro",
    "score": 0.568,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: SWE-Bench Pro (Public)' / column 'GPT-5.3-Codex (xhigh)'; value printed as '56.8%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-2-codex",
    "bench": "swe-bench-pro",
    "score": 0.564,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: SWE-Bench Pro (Public)' / column 'GPT-5.2-Codex (xhigh)'; value printed as '56.4%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "terminal-bench-2",
    "score": 0.773,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: Terminal-Bench 2.0' / column 'GPT-5.3-Codex (xhigh)'; value printed as '77.3%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-2-codex",
    "bench": "terminal-bench-2",
    "score": 0.64,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: Terminal-Bench 2.0' / column 'GPT-5.2-Codex (xhigh)'; value printed as '64.0%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Harness not stated.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "osworld-verified",
    "score": 0.647,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: OSWorld-Verified' / column 'GPT-5.3-Codex (xhigh)'; value printed as '64.7%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The later GPT-5.4 post reports GPT-5.3-Codex at 74.0% 'with a newly introduced API parameter that preserves the original image resolution' ('Previously reported as 64.7%'); this row keeps the model's own launch figure.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-2-codex",
    "bench": "osworld-verified",
    "score": 0.382,
    "source": "https://openai.com/index/introducing-gpt-5-3-codex",
    "by": "lab",
    "date": "2026-02-05",
    "note": "OpenAI 'Introducing GPT-5.3-Codex' (Feb 5, 2026), Appendix table, columns 'GPT-5.3-Codex (xhigh)', 'GPT-5.2-Codex (xhigh)'; footnote: 'All evaluations in the blog were run on GPT-5.3-Codex with xhigh reasoning effort'; read from Internet Archive capture 20260914223404 of the page (openai.com returns HTTP 403 to direct fetches). Row/label 'Appendix: OSWorld-Verified' / column 'GPT-5.2-Codex (xhigh)'; value printed as '38.2%'. Settings: reasoning effort xhigh. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The later GPT-5.4 post reports GPT-5.3-Codex at 74.0% 'with a newly introduced API parameter that preserves the original image resolution' ('Previously reported as 64.7%'); this row keeps the model's own launch figure.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "healthbench",
    "score": 0.557,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.4'; value printed as '55.7'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Comparison column in the GPT-5.6 card; the card notes comparison values may reflect later versions of earlier models.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4",
    "bench": "healthbench-hard",
    "score": 0.303,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.4'; value printed as '30.3'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Comparison column in the GPT-5.6 card; the card notes comparison values may reflect later versions of earlier models.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "healthbench",
    "score": 0.584,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.5'; value printed as '58.4'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Comparison column in the GPT-5.6 card.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5",
    "bench": "healthbench-hard",
    "score": 0.338,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.5'; value printed as '33.8'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Comparison column in the GPT-5.6 card.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "healthbench",
    "score": 0.556,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-sol'; value printed as '55.6'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "healthbench-hard",
    "score": 0.311,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-sol'; value printed as '31.1'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "healthbench",
    "score": 0.587,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-terra'; value printed as '58.7'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "healthbench-hard",
    "score": 0.343,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-terra'; value printed as '34.3'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "healthbench",
    "score": 0.554,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-luna'; value printed as '55.4'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "healthbench-hard",
    "score": 0.314,
    "source": "https://deploymentsafety.openai.com/gpt-5-6#healthbench",
    "by": "lab",
    "date": "2026-07-09",
    "note": "OpenAI 'GPT-5.6 System Card' (Deployment Safety Hub, published July 9, 2026), fetched directly. Row/label 'Table 6 (§5.1 HealthBench), 'Reported as length-adjusted score (unadjusted, mean response length in characters)': HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-5.6-luna'; value printed as '31.4'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "healthbench",
    "score": 0.569,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra#healthbench",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 6 (§6.1 HealthBench, values revised Sep 22, 2026): HealthBench length-adjusted (unadjusted score in parentheses)' / column 'gpt-6 Astra'; value printed as '56.9'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-astra",
    "bench": "healthbench-hard",
    "score": 0.342,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra#healthbench",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 6 (§6.1 HealthBench, values revised Sep 22, 2026): HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'gpt-6 Astra'; value printed as '34.2'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-sol",
    "bench": "healthbench",
    "score": 0.471,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 29 (Appendix: GPT-6 Sol, GPT-6 Luna, §11.4.1 HealthBench): HealthBench length-adjusted (unadjusted score in parentheses)' / column 'GPT-6 Sol'; value printed as '47.1'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-sol",
    "bench": "healthbench-hard",
    "score": 0.221,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 29 (Appendix: GPT-6 Sol, GPT-6 Luna, §11.4.1 HealthBench): HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'GPT-6 Sol'; value printed as '22.1'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-luna",
    "bench": "healthbench",
    "score": 0.5,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 29 (Appendix: GPT-6 Sol, GPT-6 Luna, §11.4.1 HealthBench): HealthBench length-adjusted (unadjusted score in parentheses)' / column 'GPT-6 Luna'; value printed as '50.0'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-luna",
    "bench": "healthbench-hard",
    "score": 0.254,
    "source": "https://deploymentsafety.openai.com/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-22",
    "note": "OpenAI 'GPT-6 Astra System Card' (Deployment Safety Hub; published Sep 3, 2026; HealthBench table revised and 'Appendix: GPT-6 Sol, GPT-6 Luna' added Sep 22, 2026 per its change log), fetched directly. Row/label 'Table 29 (Appendix: GPT-6 Sol, GPT-6 Luna, §11.4.1 HealthBench): HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'GPT-6 Luna'; value printed as '25.4'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "healthbench",
    "score": 0.567,
    "source": "https://deploymentsafety.openai.com/gpt-6-1-sol#healthbench",
    "by": "lab",
    "date": "2026-09-29",
    "note": "OpenAI 'Addendum to GPT-6 Astra System Card: GPT-6.1 Sol' (Deployment Safety Hub, published Sep 29, 2026), fetched directly. Row/label 'Table 7 (§5.1 HealthBench): HealthBench length-adjusted (unadjusted score in parentheses)' / column 'GPT-6.1 Sol'; value printed as '56.7'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-6-1-sol",
    "bench": "healthbench-hard",
    "score": 0.334,
    "source": "https://deploymentsafety.openai.com/gpt-6-1-sol#healthbench",
    "by": "lab",
    "date": "2026-09-29",
    "note": "OpenAI 'Addendum to GPT-6 Astra System Card: GPT-6.1 Sol' (Deployment Safety Hub, published Sep 29, 2026), fetched directly. Row/label 'Table 7 (§5.1 HealthBench): HealthBench Hard length-adjusted (unadjusted score in parentheses)' / column 'GPT-6.1 Sol'; value printed as '33.4'. Settings: reasoning effort not stated; the standard (unadjusted) score printed in parentheses is used, not the length-adjusted headline. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5",
    "bench": "simpleqa",
    "score": 0.55,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T8",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 8: SimpleQA evaluations' / row 'SimpleQA (no web) accuracy' / column 'gpt-5-thinking'; value printed as '0.55'. Settings: SimpleQA (no web), metric 'accuracy (higher is better)' (correct answers over all questions; the table's hallucination rate is the separate incorrect share); reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "o3",
    "bench": "simpleqa",
    "score": 0.54,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T8",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 8: SimpleQA evaluations' / row 'SimpleQA (no web) accuracy' / column 'OpenAI o3'; value printed as '0.54'. Settings: SimpleQA (no web), metric 'accuracy (higher is better)' (correct answers over all questions; the table's hallucination rate is the separate incorrect share); reasoning effort not stated. arXiv CC BY 4.0 (rule a). Comparison model value (latest version of the live model at the time).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "gpt-5-mini",
    "bench": "simpleqa",
    "score": 0.22,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T8",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 8: SimpleQA evaluations' / row 'SimpleQA (no web) accuracy' / column 'gpt-5-thinking-mini'; value printed as '0.22'. Settings: SimpleQA (no web), metric 'accuracy (higher is better)' (correct answers over all questions; the table's hallucination rate is the separate incorrect share); reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "o4-mini",
    "bench": "simpleqa",
    "score": 0.24,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T8",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 8: SimpleQA evaluations' / row 'SimpleQA (no web) accuracy' / column 'OpenAI o4-mini'; value printed as '0.24'. Settings: SimpleQA (no web), metric 'accuracy (higher is better)' (correct answers over all questions; the table's hallucination rate is the separate incorrect share); reasoning effort not stated. arXiv CC BY 4.0 (rule a). Comparison model value (latest version of the live model at the time).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "gpt-5-nano",
    "bench": "simpleqa",
    "score": 0.11,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T8",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 8: SimpleQA evaluations' / row 'SimpleQA (no web) accuracy' / column 'gpt-5-thinking-nano'; value printed as '0.11'. Settings: SimpleQA (no web), metric 'accuracy (higher is better)' (correct answers over all questions; the table's hallucination rate is the separate incorrect share); reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "gpt-5",
    "bench": "healthbench-hard",
    "score": 0.462,
    "source": "https://arxiv.org/html/2601.03267v2#S3.SS10",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label '§3.10 Health text ('State of the art on HealthBench Hard improves from 31.6% for OpenAI o3 to 46.2% for gpt-5-thinking ... gpt-5-thinking-mini ... 40.3% on HealthBench Hard')' / model 'gpt-5-thinking'; value printed as '46.2%'. Settings: reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "o3",
    "bench": "healthbench-hard",
    "score": 0.316,
    "source": "https://arxiv.org/html/2601.03267v2#S3.SS10",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label '§3.10 Health text ('State of the art on HealthBench Hard improves from 31.6% for OpenAI o3 to 46.2% for gpt-5-thinking ... gpt-5-thinking-mini ... 40.3% on HealthBench Hard')' / model 'OpenAI o3'; value printed as '31.6%'. Settings: reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "gpt-5-mini",
    "bench": "healthbench-hard",
    "score": 0.403,
    "source": "https://arxiv.org/html/2601.03267v2#S3.SS10",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label '§3.10 Health text ('State of the art on HealthBench Hard improves from 31.6% for OpenAI o3 to 46.2% for gpt-5-thinking ... gpt-5-thinking-mini ... 40.3% on HealthBench Hard')' / model 'gpt-5-thinking-mini'; value printed as '40.3%'. Settings: reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "gpt-5",
    "bench": "mmmlu",
    "score": 0.893571,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T11",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 11: MMLU Language (0-shot)' / column 'gpt-5-thinking' / the 14 translated languages (Arabic, Bengali, Chinese (Simplified), French, German, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese (Brazil), Spanish, Swahili, Yoruba)'; value printed as '0.903 / 0.892 / 0.902 / 0.901 / 0.896 / 0.899 / 0.909 / 0.908 / 0.898 / 0.896 / 0.910 / 0.910 / 0.880 / 0.806'. Settings: 0-shot chain-of-thought; professionally human-translated MMLU test set (OpenAI MMMLU); reasoning effort not stated. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "o3",
    "bench": "mmmlu",
    "score": 0.888429,
    "source": "https://arxiv.org/html/2601.03267v2#S3.T11",
    "by": "lab",
    "date": "2025-08-13",
    "note": "OpenAI, 'GPT-5 System Card' (dated August 13, 2025), arXiv:2601.03267v2 (v2 posted 2026-05-01), CC BY 4.0. In the card, gpt-5-thinking / -mini / -nano are the API models gpt-5 / gpt-5-mini / gpt-5-nano; 'comparison values from live models (e.g., OpenAI o3) are from the latest versions of those models'. Row/label 'Table 11: MMLU Language (0-shot)' / column 'OpenAI o3-high' / the 14 translated languages (Arabic, Bengali, Chinese (Simplified), French, German, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese (Brazil), Spanish, Swahili, Yoruba)'; value printed as '0.904 / 0.878 / 0.893 / 0.906 / 0.905 / 0.898 / 0.898 / 0.912 / 0.890 / 0.893 / 0.910 / 0.911 / 0.860 / 0.780'. Settings: 0-shot chain-of-thought; high reasoning effort ('o3-high'). arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2601.03267v2"
  },
  {
    "model": "o1-preview",
    "bench": "mmmlu",
    "score": 0.86685,
    "source": "https://arxiv.org/html/2412.16720v2#S6.T17",
    "by": "lab",
    "date": "2024-12-05",
    "note": "OpenAI, 'OpenAI o1 System Card' (dated December 5, 2024), arXiv:2412.16720v2 (v2 posted 2026-04-30), CC BY 4.0. Row/label 'Table 17: MMLU Language (0-shot)' / column 'o1-preview' / the 14 translated languages (Arabic, Bengali, Chinese (Simplified), French, German, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese (Brazil), Spanish, Swahili, Yoruba)'; value printed as '0.8821 / 0.8622 / 0.8800 / 0.8861 / 0.8573 / 0.8782 / 0.8821 / 0.8872 / 0.8788 / 0.8815 / 0.8859 / 0.8893 / 0.8479 / 0.7373'. Settings: 0-shot; MMLU test set translated into 14 languages by professional human translators (OpenAI MMMLU, simple-evals); the 'English (not translated)' row is excluded. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2412.16720v2"
  },
  {
    "model": "o1-mini",
    "bench": "mmmlu",
    "score": 0.7856,
    "source": "https://arxiv.org/html/2412.16720v2#S6.T17",
    "by": "lab",
    "date": "2024-12-05",
    "note": "OpenAI, 'OpenAI o1 System Card' (dated December 5, 2024), arXiv:2412.16720v2 (v2 posted 2026-04-30), CC BY 4.0. Row/label 'Table 17: MMLU Language (0-shot)' / column 'o1-mini' / the 14 translated languages (Arabic, Bengali, Chinese (Simplified), French, German, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese (Brazil), Spanish, Swahili, Yoruba)'; value printed as '0.7945 / 0.7725 / 0.8180 / 0.8212 / 0.8122 / 0.7887 / 0.8174 / 0.8222 / 0.8129 / 0.8020 / 0.8243 / 0.8303 / 0.7015 / 0.5807'. Settings: 0-shot; MMLU test set translated into 14 languages by professional human translators (OpenAI MMMLU, simple-evals); the 'English (not translated)' row is excluded. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2412.16720v2"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmmlu",
    "score": 0.705107,
    "source": "https://arxiv.org/html/2412.16720v2#S6.T17",
    "by": "lab",
    "date": "2024-12-05",
    "note": "OpenAI, 'OpenAI o1 System Card' (dated December 5, 2024), arXiv:2412.16720v2 (v2 posted 2026-04-30), CC BY 4.0. Row/label 'Table 17: MMLU Language (0-shot)' / column 'GPT-4o-mini' / the 14 translated languages (Arabic, Bengali, Chinese (Simplified), French, German, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese (Brazil), Spanish, Swahili, Yoruba)'; value printed as '0.7089 / 0.6577 / 0.7305 / 0.7659 / 0.7431 / 0.6916 / 0.7452 / 0.7640 / 0.7255 / 0.7203 / 0.7677 / 0.7737 / 0.6191 / 0.4583'. Settings: 0-shot; MMLU test set translated into 14 languages by professional human translators (OpenAI MMMLU, simple-evals); the 'English (not translated)' row is excluded. arXiv CC BY 4.0 (rule a). 'GPT-4o-mini' mapped to gpt-4o-mini-2024-07-18, the only GPT-4o mini snapshot released before the card (Dec 2024).",
    "source_id": "paper-2412.16720v2"
  },
  {
    "model": "o1-preview",
    "bench": "swe-bench-verified",
    "score": 0.413,
    "source": "https://arxiv.org/html/2412.16720v2#S5.SS8.SSS2",
    "by": "lab",
    "date": "2024-12-05",
    "note": "OpenAI, 'OpenAI o1 System Card' (dated December 5, 2024), arXiv:2412.16720v2 (v2 posted 2026-04-30), CC BY 4.0. Row/label '§5.8.2 SWE-bench Verified text ('o1-preview (pre-mitigation and post-mitigation) performs the best on SWE-bench Verified at 41.3%')'; value printed as '41.3%'. Settings: Agentless scaffold; 5 tries per instance, pass@1 = mean per-instance pass rate over valid patches; full 500-task Verified set. arXiv CC BY 4.0 (rule a).",
    "source_id": "paper-2412.16720v2"
  },
  {
    "model": "stt-en-conformer-ctc-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.037,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_ctc_small', column 'Librispeech Test-Clean' = 3.7 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.6.0, LS test-clean 3.7, test-other 8.1 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-conformer-ctc-small",
    "bench": "librispeech-test-other-wer",
    "score": 0.081,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_ctc_small', column 'Librispeech Test-Other' = 8.1 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.6.0, LS test-clean 3.7, test-other 8.1 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-conformer-transducer-large",
    "bench": "librispeech-test-clean-wer",
    "score": 0.017,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_transducer_large', column 'Librispeech Test-Clean' = 1.7 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.10.0, LS test-clean 1.7, test-other 3.7 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-conformer-transducer-large",
    "bench": "librispeech-test-other-wer",
    "score": 0.037,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_transducer_large', column 'Librispeech Test-Other' = 3.7 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.10.0, LS test-clean 1.7, test-other 3.7 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-conformer-transducer-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.025,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_transducer_small', column 'Librispeech Test-Clean' = 2.5 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: NGC-only checkpoint (no HF card); the NeMo pretrained name stt_en_conformer_transducer_small is the catalogue identity.",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-conformer-transducer-small",
    "bench": "librispeech-test-other-wer",
    "score": 0.066,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/conformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file conformer_en.csv, row 'stt_en_conformer_transducer_small', column 'Librispeech Test-Other' = 6.6 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: NGC-only checkpoint (no HF card); the NeMo pretrained name stt_en_conformer_transducer_small is the catalogue identity.",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-fastconformer-ctc-large",
    "bench": "librispeech-test-clean-wer",
    "score": 0.021,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/fastconformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file fastconformer_en.csv, row 'stt_en_fastconformer_ctc_large', column 'Librispeech Test-Clean' = 2.1 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.18.0, LS test-clean 2.1, test-other 4.2 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-fastconformer-ctc-large",
    "bench": "librispeech-test-other-wer",
    "score": 0.042,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/fastconformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file fastconformer_en.csv, row 'stt_en_fastconformer_ctc_large', column 'Librispeech Test-Other' = 4.2 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.18.0, LS test-clean 2.1, test-other 4.2 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-fastconformer-transducer-large",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/fastconformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file fastconformer_en.csv, row 'stt_en_fastconformer_transducer_large', column 'Librispeech Test-Clean' = 1.8 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.18.0, LS test-clean 1.8, test-other 3.8 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "stt-en-fastconformer-transducer-large",
    "bench": "librispeech-test-other-wer",
    "score": 0.038,
    "source": "https://github.com/NVIDIA/NeMo/blob/6f20e61e4d11ce4dafc0f037c3bad535e1ac3ea9/docs/source/asr/data/scores/en/fastconformer_en.csv",
    "by": "lab",
    "date": "2023-07-22",
    "note": "NVIDIA NeMo docs ASR 'Scores' table (docs/source/asr/scores.rst renders docs/source/asr/data/scores/en/*.csv), file fastconformer_en.csv, row 'stt_en_fastconformer_transducer_large', column 'Librispeech Test-Other' = 3.8 (WER %). NVIDIA's own checkpoints scored by NVIDIA; greedy decoding per the model card. Only LibriSpeech test-clean/test-other recorded (dev sets not). Identity: HF card version table: 1.18.0, LS test-clean 1.8, test-other 3.8 (same as this row).",
    "source_id": "speech-nemo-asr-scores-apache"
  },
  {
    "model": "whisper-small",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.508,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper small', column FLEURS-81 test = 50.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-small",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 1.117,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper small', column CV22-76 test = 111.7 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-medium",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.478,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper medium', column FLEURS-81 test = 47.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-medium",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.879,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper medium', column CV22-76 test = 87.9 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-large-v3",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.226,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper large-v3', column FLEURS-81 test = 22.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-large-v3",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.556,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "independent",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row 'Whisper large-v3', column CV22-76 test = 55.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). OpenAI Whisper checkpoint, decoding settings not stated.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-300m",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.118,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '300M CTC', column FLEURS-81 test = 11.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-300m",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.176,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '300M CTC', column CV22-76 test = 17.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-1b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.086,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '1B CTC', column FLEURS-81 test = 8.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-1b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.148,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '1B CTC', column CV22-76 test = 14.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-3b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.078,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '3B CTC', column FLEURS-81 test = 7.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-3b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.137,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '3B CTC', column CV22-76 test = 13.7 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-7b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.073,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '7B CTC', column FLEURS-81 test = 7.3 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-ctc-7b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.138,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '7B CTC', column CV22-76 test = 13.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-300m",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.078,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '300M LLM-ASR', column FLEURS-81 test = 7.8 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-300m",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.071,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '300M LLM-ASR', column CV22-76 test = 7.1 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-1b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.066,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '1B LLM-ASR', column FLEURS-81 test = 6.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-1b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.065,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '1B LLM-ASR', column CV22-76 test = 6.5 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-3b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.062,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '3B LLM-ASR', column FLEURS-81 test = 6.2 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-3b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.066,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '3B LLM-ASR', column CV22-76 test = 6.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "omniasr-paper-fleurs81-test-cer",
    "score": 0.056,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '7B LLM-ASR', column FLEURS-81 test = 5.6 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "omniasr-llm-7b",
    "bench": "omniasr-paper-cv22-76-test-cer",
    "score": 0.064,
    "source": "https://arxiv.org/html/2511.09690v1",
    "by": "lab",
    "date": "2025-11-12",
    "note": "Omnilingual ASR paper (CC BY-SA 4.0) Table 5 'Comparison against Whisper v3', row '7B LLM-ASR', column CV22-76 test = 6.4 (average CER % across the Whisper-covered languages of the set). Authors' own run of all rows (they restrict each set to languages Whisper covers); Omnilingual rows without LM fusion ('7B LLM-ASR + LM' row not recorded). Omnilingual first-release (v1, Nov 2025) checkpoint; the _v2 checkpoints came out in Dec 2025 and are not in this paper.",
    "source_id": "speech-omniasr-paper-bysa"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0403,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v3.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.03% at Q8_0 (95% CI 3.59-4.46; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.81% at F16 vs 1.82% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "fleurs-en-wer",
    "score": 0.0438,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3-turbo.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v3-turbo.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.38% at Q8_0 (95% CI 3.95-4.84; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.01% at F16 vs 2.01% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "fleurs-en-wer",
    "score": 0.0411,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-0.6b-v2.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt-0.6b-v2.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.11% at Q8_0 (95% CI 3.71-4.53; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.68% at F32 vs 1.69% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0483,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-0.6b-v3.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt-0.6b-v3.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.83% at Q8_0 (95% CI 4.39-5.3; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.95% at F32 vs 1.94% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-rnnt-1-1b",
    "bench": "fleurs-en-wer",
    "score": 0.0445,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-rnnt-1.1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-rnnt-1.1b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.45% at Q8_0 (95% CI 3.99-4.91; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.45% at F32 vs 1.46% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "canary-1b-v2",
    "bench": "fleurs-en-wer",
    "score": 0.0447,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b-v2.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/canary-1b-v2.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.47% at Q8_0 (95% CI 4.03-4.95; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.92% at F32 vs 1.91% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "canary-qwen-2-5b",
    "bench": "fleurs-en-wer",
    "score": 0.0358,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-qwen-2.5b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/canary-qwen-2.5b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 3.58% at Q8_0 (95% CI 3.19-4.0; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.63% at BF16 vs 1.63% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "fleurs-en-wer",
    "score": 0.0508,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/cohere-transcribe-03-2026.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/cohere-transcribe-03-2026.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 5.08% at Q8_0 (95% CI 4.56-5.59; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.26% at BF16 vs 1.27% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-4-1-2b",
    "bench": "fleurs-en-wer",
    "score": 0.0414,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-speech-4.1-2b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.14% at Q8_0 (95% CI 3.68-4.62; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.31% at BF16 vs 1.32% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-5-0-470m-turboctc",
    "bench": "fleurs-en-wer",
    "score": 0.0461,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-speech-5.0-470m-turboctc.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-5.0-470m-turboctc.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.61% at Q8_0 (95% CI 4.19-5.07; batch_size 8, timestamps none, provenance 'None'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.34% at BF16 vs 1.33% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0423,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/qwen3-asr-0.6b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.23% at Q8_0 (95% CI 3.76-4.69; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.12% at BF16 vs 2.11% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "fleurs-en-wer",
    "score": 0.0323,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-1.7b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/qwen3-asr-1.7b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 3.23% at Q8_0 (95% CI 2.85-3.65; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.62% at BF16 vs 1.62% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "fleurs-en-wer",
    "score": 0.0389,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-mini-3b-2507.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/voxtral-mini-3b-2507.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 3.89% at Q8_0 (95% CI 3.47-4.35; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.88% at BF16 vs 1.87% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "fleurs-en-wer",
    "score": 0.0355,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-small-24b-2507.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/voxtral-small-24b-2507.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 3.55% at Q8_0 (95% CI 3.07-4.1; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.56% at BF16 vs 1.56% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "fleurs-en-wer",
    "score": 0.0627,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-mini-4b-realtime-2602.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/voxtral-mini-4b-realtime-2602.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 6.27% at Q8_0 (95% CI 5.38-7.31; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.08% at BF16 vs 2.07% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-tiny",
    "bench": "fleurs-en-wer",
    "score": 0.1384,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-tiny.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-tiny.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 13.84% at Q8_0 (95% CI 12.85-14.94; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 7.49% at F32 vs 7.52% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-tiny-en",
    "bench": "fleurs-en-wer",
    "score": 0.1072,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-tiny.en.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-tiny.en.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 10.72% at Q8_0 (95% CI 9.89-11.58; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 5.77% at F32 vs 5.72% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-base",
    "bench": "fleurs-en-wer",
    "score": 0.0988,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-base.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-base.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 9.88% at Q8_0 (95% CI 9.07-10.7; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 5.11% at F32 vs 5.12% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-base-en",
    "bench": "fleurs-en-wer",
    "score": 0.076,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-base.en.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-base.en.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 7.6% at Q8_0 (95% CI 6.89-8.32; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 4.3% at F32 vs 4.16% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-small",
    "bench": "fleurs-en-wer",
    "score": 0.0651,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-small.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 6.51% at Q8_0 (95% CI 5.88-7.22; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 3.34% at F32 vs 3.33% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-small-en",
    "bench": "fleurs-en-wer",
    "score": 0.0614,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-small.en.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-small.en.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 6.14% at Q8_0 (95% CI 5.47-6.87; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 3.11% at F32 vs 3.09% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-medium",
    "bench": "fleurs-en-wer",
    "score": 0.0464,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-medium.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-medium.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.64% at Q8_0 (95% CI 4.2-5.15; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.64% at F32 vs 2.64% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-medium-en",
    "bench": "fleurs-en-wer",
    "score": 0.0488,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-medium.en.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-medium.en.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.88% at Q8_0 (95% CI 4.32-5.56; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.74% at F32 vs 2.72% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large",
    "bench": "fleurs-en-wer",
    "score": 0.0446,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.46% at Q8_0 (95% CI 3.9-5.11; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.72% at F32 vs 2.71% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v2",
    "bench": "fleurs-en-wer",
    "score": 0.0421,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v2.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v2.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.21% at Q8_0 (95% CI 3.73-4.82; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.67% at F32 vs 2.97% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "canary-1b",
    "bench": "fleurs-en-wer",
    "score": 0.0444,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/canary-1b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.44% at Q8_0 (95% CI 4.0-4.89; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.55% at F32 vs 1.55% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "canary-1b-flash",
    "bench": "fleurs-en-wer",
    "score": 0.0475,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-1b-flash.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/canary-1b-flash.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.75% at Q8_0 (95% CI 4.14-5.51; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.62% at F32 vs 1.62% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "canary-180m-flash",
    "bench": "fleurs-en-wer",
    "score": 0.0598,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/canary-180m-flash.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/canary-180m-flash.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 5.98% at Q8_0 (95% CI 5.26-6.78; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.94% at F32 vs 1.93% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-ctc-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0553,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-ctc-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-ctc-0.6b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 5.53% at Q8_0 (95% CI 5.06-6.07; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.87% at F32 vs 1.87% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-ctc-1-1b",
    "bench": "fleurs-en-wer",
    "score": 0.0561,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-ctc-1.1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-ctc-1.1b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 5.61% at Q8_0 (95% CI 5.04-6.21; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.85% at F32 vs 1.85% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-rnnt-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0457,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-rnnt-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-rnnt-0.6b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.57% at Q8_0 (95% CI 4.13-5.05; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.62% at F32 vs 1.62% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-1-1b",
    "bench": "fleurs-en-wer",
    "score": 0.0424,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt-1.1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt-1.1b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.24% at Q8_0 (95% CI 3.78-4.71; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.39% at F32 vs 1.38% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-ctc-110m",
    "bench": "fleurs-en-wer",
    "score": 0.0611,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/parakeet-tdt_ctc-110m.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt_ctc-110m.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 6.11% at Q8_0 (95% CI 5.59-6.66; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.43% at F32 vs 2.43% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-4-0-1b-speech",
    "bench": "fleurs-en-wer",
    "score": 0.0466,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/granite-4.0-1b-speech.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-4.0-1b-speech.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.66% at Q8_0 (95% CI 4.2-5.15; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.42% at BF16 vs 1.44% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moonshine-tiny",
    "bench": "fleurs-en-wer",
    "score": 0.1413,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-tiny.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moonshine-tiny.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 14.13% at Q8_0 (95% CI 12.44-16.27; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 4.58% at F32 vs 4.6% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moonshine-base",
    "bench": "fleurs-en-wer",
    "score": 0.1225,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-base.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moonshine-base.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 12.25% at Q8_0 (95% CI 10.67-14.04; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 3.28% at F32 vs 3.26% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moonshine-streaming-tiny",
    "bench": "fleurs-en-wer",
    "score": 0.1818,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-tiny.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moonshine-streaming-tiny.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 18.18% at Q8_0 (95% CI 15.79-22.01; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 4.53% at F32 vs 4.52% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moonshine-streaming-small",
    "bench": "fleurs-en-wer",
    "score": 0.0855,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moonshine-streaming-small.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 8.55% at Q8_0 (95% CI 7.84-9.34; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.53% at F32 vs 2.54% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moonshine-streaming-medium",
    "bench": "fleurs-en-wer",
    "score": 0.0787,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/moonshine-streaming-medium.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moonshine-streaming-medium.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 7.87% at Q8_0 (95% CI 6.97-8.85; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 2.16% at F32 vs 2.16% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "cohere-transcribe-arabic-07-2026",
    "bench": "fleurs-en-wer",
    "score": 0.0488,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/cohere-transcribe-arabic-07-2026.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/cohere-transcribe-arabic-07-2026.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 4.88% at Q8_0 (95% CI 4.25-5.61; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 1.33% at BF16 vs 1.34% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.079,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/nemotron-3.5-asr-streaming-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/nemotron-3.5-asr-streaming-0.6b.json accuracy_benchmarks: FLEURS test, language en (647 utt.), corpus WER 7.9% at Q8_0 (95% CI 7.23-8.54; batch_size 8, timestamps none, provenance 'legacy-published'). FLEURS is benchmarked only at Q8_0 (8-bit GGUF of the same upstream checkpoint) in the catalog's asr-publication-v2 profile; for this model LibriSpeech test-clean is 3.04% at F32 vs 3.05% at Q8_0, so Q8_0 tracks full precision closely. Same normaliser/recipe as the existing LibriSpeech rows (whisper_normalizer EnglishTextNormalizer + jiwer). Nemotron also has F32 7.97% (batch 1); Q8_0 kept for a uniform run.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v3",
    "bench": "distil-whisper-ood4-shortform-wer",
    "score": 0.084,
    "source": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md",
    "by": "independent",
    "date": "2024-03-21",
    "note": "Distil-Whisper GitHub README (MIT) headline table, row 'large-v3', column 'Short-Form WER' = 8.4%: macro-average WER over the four out-of-distribution short-form test sets of the Distil-Whisper paper (CHiME-4, Earnings-22, FLEURS, SPGISpeech). The README's distil-medium.en (11.1) and distil-large-v2 (10.1) values equal the paper's Table 5 short-form averages, confirming the protocol. Long-Form column not recorded (it mixes sequential and chunked decoding across rows).",
    "source_id": "speech-distil-whisper-readme-mit"
  },
  {
    "model": "distil-large-v3",
    "bench": "distil-whisper-ood4-shortform-wer",
    "score": 0.097,
    "source": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md",
    "by": "lab",
    "date": "2024-03-21",
    "note": "Distil-Whisper GitHub README (MIT) headline table, row 'distil-large-v3', column 'Short-Form WER' = 9.7%: macro-average WER over the four out-of-distribution short-form test sets of the Distil-Whisper paper (CHiME-4, Earnings-22, FLEURS, SPGISpeech). The README's distil-medium.en (11.1) and distil-large-v2 (10.1) values equal the paper's Table 5 short-form averages, confirming the protocol. Long-Form column not recorded (it mixes sequential and chunked decoding across rows).",
    "source_id": "speech-distil-whisper-readme-mit"
  },
  {
    "model": "distil-large-v2",
    "bench": "distil-whisper-ood4-shortform-wer",
    "score": 0.101,
    "source": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md",
    "by": "lab",
    "date": "2024-03-21",
    "note": "Distil-Whisper GitHub README (MIT) headline table, row 'distil-large-v2', column 'Short-Form WER' = 10.1%: macro-average WER over the four out-of-distribution short-form test sets of the Distil-Whisper paper (CHiME-4, Earnings-22, FLEURS, SPGISpeech). The README's distil-medium.en (11.1) and distil-large-v2 (10.1) values equal the paper's Table 5 short-form averages, confirming the protocol. Long-Form column not recorded (it mixes sequential and chunked decoding across rows).",
    "source_id": "speech-distil-whisper-readme-mit"
  },
  {
    "model": "distil-medium-en",
    "bench": "distil-whisper-ood4-shortform-wer",
    "score": 0.111,
    "source": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md",
    "by": "lab",
    "date": "2024-03-21",
    "note": "Distil-Whisper GitHub README (MIT) headline table, row 'distil-medium.en', column 'Short-Form WER' = 11.1%: macro-average WER over the four out-of-distribution short-form test sets of the Distil-Whisper paper (CHiME-4, Earnings-22, FLEURS, SPGISpeech). The README's distil-medium.en (11.1) and distil-large-v2 (10.1) values equal the paper's Table 5 short-form averages, confirming the protocol. Long-Form column not recorded (it mixes sequential and chunked decoding across rows).",
    "source_id": "speech-distil-whisper-readme-mit"
  },
  {
    "model": "distil-small-en",
    "bench": "distil-whisper-ood4-shortform-wer",
    "score": 0.121,
    "source": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md",
    "by": "lab",
    "date": "2024-03-21",
    "note": "Distil-Whisper GitHub README (MIT) headline table, row 'distil-small.en', column 'Short-Form WER' = 12.1%: macro-average WER over the four out-of-distribution short-form test sets of the Distil-Whisper paper (CHiME-4, Earnings-22, FLEURS, SPGISpeech). The README's distil-medium.en (11.1) and distil-large-v2 (10.1) values equal the paper's Table 5 short-form averages, confirming the protocol. Long-Form column not recorded (it mixes sequential and chunked decoding across rows).",
    "source_id": "speech-distil-whisper-readme-mit"
  },
  {
    "model": "whisper-large-v3",
    "bench": "mls-fr-wer",
    "score": 0.072,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "independent",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'Original', column FR (MLS French) = 7.2 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "whisper-large-v3",
    "bench": "mls-de-wer",
    "score": 0.132,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "independent",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'Original', column DE (MLS German) = 13.2 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3-acc",
    "bench": "mls-fr-wer",
    "score": 0.074,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (a)', column FR (MLS French) = 7.4 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3-acc",
    "bench": "mls-de-wer",
    "score": 0.087,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (a)', column DE (MLS German) = 8.7 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3",
    "bench": "mls-fr-wer",
    "score": 0.068,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (b)', column FR (MLS French) = 6.8 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3",
    "bench": "mls-de-wer",
    "score": 0.077,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (b)', column DE (MLS German) = 7.7 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3-fast",
    "bench": "mls-fr-wer",
    "score": 0.091,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (c)', column FR (MLS French) = 9.1 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "lite-whisper-large-v3-fast",
    "bench": "mls-de-wer",
    "score": 0.101,
    "source": "https://arxiv.org/html/2502.20583v2",
    "by": "lab",
    "date": "2025-02-27",
    "note": "LiteASR (Kamahori et al., arXiv:2502.20583v2, CC BY 4.0) Table 2 'Sensitivity study on other languages', row 'LiteASR (c)', column DE (MLS German) = 10.1 WER %. §4.4.3: MLS test for French and German; Whisper large-v3 compressed with the same English calibration data as Table 1, so (a)/(b)/(c) are the same configurations as the Table 1 rows already mapped to lite-whisper-large-v3-acc / -large-v3 / -fast. Authors' own runs; JSUT Japanese CER column not recorded.",
    "source_id": "speech-stt-liteasr-report-ccby"
  },
  {
    "model": "canary-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.017,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'cnry-1b', column 'clean (WER)': 1.7% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-medium",
    "bench": "librispeech-test-clean-wer",
    "score": 0.039,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-md', column 'clean (WER)': 3.9% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-large-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.039,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-lg', column 'clean (WER)': 3.9% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.043,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-sm', column 'clean (WER)': 4.3% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-base",
    "bench": "librispeech-test-clean-wer",
    "score": 0.059,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-bs', column 'clean (WER)': 5.9% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'w2v2-lg-slf', column 'clean (WER)': 1.8% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-tiny-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.064,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-tn.en', column 'clean (WER)': 6.4% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-tiny",
    "bench": "librispeech-test-clean-wer",
    "score": 0.082,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-tn', column 'clean (WER)': 8.2% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "hubert-large-ls960-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.021,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'hubt-lg', column 'clean (WER)': 2.1% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "hubert-xlarge-ls960-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.021,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'hubt-xl', column 'clean (WER)': 2.1% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "wav2vec2-large-robust-ft-libri-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.026,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'w2v2-lg-rob', column 'clean (WER)': 2.6% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.03,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'w2v2-lg', column 'clean (WER)': 3.0% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "wav2vec2-base-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.037,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'w2v2-bs', column 'clean (WER)': 3.7% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "mms-1b-fl102",
    "bench": "librispeech-test-clean-wer",
    "score": 0.154,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'mms-1b', column 'clean (WER)': 15.4% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-small-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.04,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-sm.en', column 'clean (WER)': 4.0% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-medium-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.041,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-md.en', column 'clean (WER)': 4.1% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "whisper-base-en",
    "bench": "librispeech-test-clean-wer",
    "score": 0.051,
    "source": "https://arxiv.org/html/2403.07937v3",
    "by": "independent",
    "date": "2024-12-09",
    "note": "Speech Robust Bench v3 Table 7, language 'en', dataset 'LibriSpeech', row 'wsp-bs.en', column 'clean (WER)': 5.1% on LibriSpeech test-clean (2,620 utt., Table 4). Authors' own zero-shot runs of Hugging Face checkpoints (§4.1, Table 5 names the checkpoints). Only the clean-speech WER column is used; the perturbation columns are NWERD/WERD degradations. Same evaluator and abbreviation mapping as the existing tedlium-wer rows.",
    "source_id": "speech-stt-srb-report"
  },
  {
    "model": "vosk-en-us-0-22",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0569,
    "source": "https://github.com/alphacep/vosk-space/blob/f10f63bc6d751cf5c4198a7dc88515f40f2d7020/models.md",
    "by": "lab",
    "date": "2026-05-04",
    "note": "Alpha Cephei Vosk models page (models.md, rendered at https://alphacephei.com/vosk/models), English table, row 'vosk-model-en-us-0.22', column 'Word error rate/Speed': '5.69 (librispeech test-clean)'. Vendor-reported (rule b), hand-recorded; decoding/normalisation not stated (Kaldi model with its bundled graph).",
    "source_id": "first-party-alphacephei-vosk-models"
  },
  {
    "model": "vosk-en-us-0-22",
    "bench": "tedlium-wer",
    "score": 0.0605,
    "source": "https://github.com/alphacep/vosk-space/blob/f10f63bc6d751cf5c4198a7dc88515f40f2d7020/models.md",
    "by": "lab",
    "date": "2026-05-04",
    "note": "Alpha Cephei Vosk models page, English table, row 'vosk-model-en-us-0.22', column 'Word error rate/Speed': '6.05 (tedlium)'. TED-LIUM release and split not stated by the page (assumed the standard test set). Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-alphacephei-vosk-models"
  },
  {
    "model": "vosk-small-en-us-0-15",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0985,
    "source": "https://github.com/alphacep/vosk-space/blob/f10f63bc6d751cf5c4198a7dc88515f40f2d7020/models.md",
    "by": "lab",
    "date": "2026-05-04",
    "note": "Alpha Cephei Vosk models page, English table, row 'vosk-model-small-en-us-0.15', column 'Word error rate/Speed': '9.85 (librispeech test-clean)'. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-alphacephei-vosk-models"
  },
  {
    "model": "vosk-small-en-us-0-15",
    "bench": "tedlium-wer",
    "score": 0.1038,
    "source": "https://github.com/alphacep/vosk-space/blob/f10f63bc6d751cf5c4198a7dc88515f40f2d7020/models.md",
    "by": "lab",
    "date": "2026-05-04",
    "note": "Alpha Cephei Vosk models page, English table, row 'vosk-model-small-en-us-0.15', column 'Word error rate/Speed': '10.38 (tedlium)'. TED-LIUM release and split not stated by the page. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-alphacephei-vosk-models"
  },
  {
    "model": "owsm-ctc-v3-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0236,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report (ESPnet / CMU WAVLab, arXiv:2506.00338v1) Table 7 'English ASR WERs (%) on the Hugging Face Open ASR Leaderboard' test sets, row 'OWSM-CTC v3.1', column 'LS-Clean' = 2.36. Authors decode their own CTC models with ESPnet (footnote 6). Vendor-reported (rule b), hand-recorded; other labs' rows not used.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "owsm-ctc-v3-1-1b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0512,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report Table 7, row 'OWSM-CTC v3.1', column 'LS-Other' = 5.12. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "owsm-ctc-v3-2-ft-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0261,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report Table 7, row 'OWSM-CTC v3.2', column 'LS-Clean' = 2.61. Identity: Table 4 footnote ‡ 'Trained on v3.1 data and then fine-tuned on v3.2' and §2.1 footnote 2 links https://huggingface.co/espnet/owsm_ctc_v3.2_ft_1B. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "owsm-ctc-v3-2-ft-1b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0532,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report Table 7, row 'OWSM-CTC v3.2', column 'LS-Other' = 5.32. Identity as for the LS-Clean row (owsm_ctc_v3.2_ft_1B). Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "owsm-ctc-v4-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0256,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report Table 7, row 'OWSM-CTC v4 (ours)', column 'LS-Clean' = 2.56. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "owsm-ctc-v4-1b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0486,
    "source": "https://arxiv.org/html/2506.00338v1",
    "by": "lab",
    "date": "2025-05-31",
    "note": "OWSM v4 report Table 7, row 'OWSM-CTC v4 (ours)', column 'LS-Other' = 4.86. Vendor-reported (rule b), hand-recorded.",
    "source_id": "first-party-espnet-owsm-v4-report"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.05,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13, LibriTTS block, row 'CosyVoice', column WER = 0.05 ± 0.00 (fraction as printed, Whisper-medium transcription, lowercase/punctuation-stripped). Replaces the v2 value 0.17 (row 'CozyVoice'). Same run and identity mapping as the rvcbench-libritts-ecapa-sim row. Re-read 2026-10-11.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "spark-tts-0-5b",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.33,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'SparkAudio', column WER = 0.33 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.41 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "higgs-audio-v2",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.25,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'Higgs', column WER = 0.25 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.56 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "vibevoice-1-5b",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.23,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'VibeVoice', column WER = 0.23 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.48 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "mgm-omni-tts-2b-0927",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.09,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'MGM-Omni', column WER = 0.09 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.54 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.05,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'Qwen-TTS', column WER = 0.05 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.61 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "f5-tts-v1-base",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.12,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'F5-TTS', column WER = 0.12 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.56 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "maskgct",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.09,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'MaskGCT', column WER = 0.09 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.57 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "openvoice-v2",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.07,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'OpenVoice', column WER = 0.07 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.24 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "xtts-v2",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.07,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'XTTS', column WER = 0.07 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.45 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "zipvoice",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.05,
    "source": "https://arxiv.org/html/2602.00443v2",
    "by": "independent",
    "date": "2026-05-24",
    "note": "RVCBench v2 Table 13 (Appendix G.1), LibriTTS block, row 'ZipVoice', column WER = 0.05 (fraction, 1000-sample bootstrap ±). Whisper-medium transcription, lowercase/punctuation-stripped (Appendix C.1). Same run and identity mapping as the existing rvcbench-libritts-ecapa-sim row (SIM 0.58 cross-checked).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "melotts-english",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.0464,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'MeloTTS† Zhao et al. (2023)', column 'Whisper-WER' = 4.64 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 3.80 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "parler-tts-mini-v1",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.0466,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'Parler-TTS† Lacombe et al. (2024)', column 'Whisper-WER' = 4.66 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 3.83 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "cosyvoice-300m",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.0495,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'CosyVoice† Du et al. (2024)', column 'Whisper-WER' = 4.95 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 4.15 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "vall-e-x-plachtaa",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.2733,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'VallE-X Zhang et al. (2023c)', column 'Whisper-WER' = 27.33 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 3.38 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "amphion-valle-v2",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.2783,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'Vall-E 2 Chen et al. (2024b)', column 'Whisper-WER' = 27.83 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 3.65 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "amphion-naturalspeech2",
    "bench": "versa-librispeech-test-clean-whisper-wer",
    "score": 0.0462,
    "source": "https://arxiv.org/html/2412.17667v2",
    "by": "independent",
    "date": "2025-03-27",
    "note": "VERSA (arXiv:2412.17667v2, CC BY 4.0) Appendix C.2 Table 6, row 'NaturalSpeech2 Shen et al. (2023)', column 'Whisper-WER' = 4.62 (%; ground truth 2.59). Same run and identity as the existing versa-librispeech-test-clean-utmos row (UTMOS 2.35 cross-checked).",
    "source_id": "speech-versa-toolkit-ccby"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.6536,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, Seed-TTS-Eval table, test-en SIM: first printed value 65.36 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: CosyVoice2-0.5B (cosyvoice2-vc config: FunAudioLLM/CosyVoice2-0.5B). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7503,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, Seed-TTS-Eval table, test-zh SIM: first printed value 75.03 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: CosyVoice2-0.5B (cosyvoice2-vc config: FunAudioLLM/CosyVoice2-0.5B). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.6945,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, Seed-TTS-Eval table, test-en SIM: first printed value 69.45 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: cosyvoice3-latest config: FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint with official CosyVoice code (not the _RL variant). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7755,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, Seed-TTS-Eval table, test-zh SIM: first printed value 77.55 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: cosyvoice3-latest config: FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint with official CosyVoice code (not the _RL variant). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.704,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, Seed-TTS-Eval table, test-en SIM: first printed value 70.40 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: IndexTeam/IndexTTS-2. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7604,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, Seed-TTS-Eval table, test-zh SIM: first printed value 76.04 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: IndexTeam/IndexTTS-2. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.7323,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, Seed-TTS-Eval table, test-en SIM: first printed value 73.23 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: openbmb/VoxCPM-0.5B (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.771,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, Seed-TTS-Eval table, test-zh SIM: first printed value 77.10 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: openbmb/VoxCPM-0.5B (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.75253,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, Seed-TTS-Eval table, test-en SIM: first printed value 75.253 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: openbmb/VoxCPM2, cfg_value=2.0, inference_timesteps=10, no denoise (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.79403,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, Seed-TTS-Eval table, test-zh SIM: first printed value 79.403 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: openbmb/VoxCPM2, cfg_value=2.0, inference_timesteps=10, no denoise (OpenBMB evaluating its own model). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.6494,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
    "by": "independent",
    "date": "2026-04-01",
    "note": "UltraEval-Audio replication/fishaudio-s2-pro.md, Seed-TTS-Eval table, test-en SIM: first printed value 64.94 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: open weights fishaudio/s2-pro via fish-speech local inference (doc notes paper numbers used the hosted service). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7284,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
    "by": "independent",
    "date": "2026-04-01",
    "note": "UltraEval-Audio replication/fishaudio-s2-pro.md, Seed-TTS-Eval table, test-zh SIM: first printed value 72.84 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: open weights fishaudio/s2-pro via fish-speech local inference (doc notes paper numbers used the hosted service). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.6795,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, Seed-TTS-Eval table, test-en SIM: first printed value 67.95 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), bfloat16. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7613,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, Seed-TTS-Eval table, test-zh SIM: first printed value 76.13 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), bfloat16. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.7124,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-en SIM: first printed value 71.24 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (the xvec_only row is not used). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7689,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-zh SIM: first printed value 76.89 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (the xvec_only row is not used). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.7055,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-en SIM: first printed value 70.55 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7648,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, Seed-TTS-Eval table, test-zh SIM: first printed value 76.48 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params. Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fireredtts3-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.772,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
    "by": "independent",
    "date": "2026-08-22",
    "note": "UltraEval-Audio replication/FireRedTTS3.md, Seed-TTS-Eval table, test-en SIM: first printed value 77.20 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-en-whisper-v3-wer row. Identity: FireRedTeam/FireRedTTS3, mode base (zero-shot cloning). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fireredtts3-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.8077,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
    "by": "independent",
    "date": "2026-08-22",
    "note": "UltraEval-Audio replication/FireRedTTS3.md, Seed-TTS-Eval table, test-zh SIM: first printed value 80.77 (OpenBMB's reproduced measurement; any parenthesised official value NOT used). SIM scorer: audio_evals/lib/simo with the WavLM-large speaker-verification checkpoint of seed-tts-eval (registry/model/wavlm.yaml). Same run and identity as the existing seed-tts-zh-paraformer-cer row. Identity: FireRedTeam/FireRedTTS3, mode base (zero-shot cloning). Dataset TwinkStart/Seed-TTS-Eval; scorer registry/evaluator/seed_tts_eval.yaml.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-asr-nano-2512",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.0673,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "lab",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'GLM-ASR-Nano (1.5B)', row 'Wenet Meeting' = 6.73 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used). Wenet Meeting = WenetSpeech test_meeting (Mandarin, character error).",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "glm-asr-nano-2512",
    "bench": "librispeech-test-clean-wer",
    "score": 0.02,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "lab",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'GLM-ASR-Nano (1.5B)', row 'Libri Clean' = 2.00 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used). ",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "glm-asr-nano-2512",
    "bench": "librispeech-test-other-wer",
    "score": 0.0419,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "lab",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'GLM-ASR-Nano (1.5B)', row 'Libri Other' = 4.19 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used). ",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "glm-asr-nano-2512",
    "bench": "fleurs-en-wer",
    "score": 0.0578,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "lab",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'GLM-ASR-Nano (1.5B)', row 'Fleurs EN' = 5.78 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used). ",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "whisper-large-v3",
    "bench": "wenetspeech-test-meeting-err",
    "score": 0.1839,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "independent",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'OpenAI Whisper V3 (1.5B)', row 'Wenet Meeting' = 18.39 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used). Wenet Meeting = WenetSpeech test_meeting (Mandarin, character error). Whisper V3 = openai/whisper-large-v3 as run by Z.ai; decoding not stated.",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0186,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "independent",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'OpenAI Whisper V3 (1.5B)', row 'Libri Clean' = 1.86 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used).  Whisper V3 = openai/whisper-large-v3 as run by Z.ai; decoding not stated.",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0343,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "independent",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'OpenAI Whisper V3 (1.5B)', row 'Libri Other' = 3.43 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used).  Whisper V3 = openai/whisper-large-v3 as run by Z.ai; decoding not stated.",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0623,
    "source": "https://github.com/zai-org/GLM-ASR/blob/a324aa59a30952c54e8097c85efea592b7fafd81/resources/bench.png",
    "by": "independent",
    "date": "2025-12-09",
    "note": "Z.ai GLM-ASR README benchmark table (image resources/bench.png, Apache-2.0 repo), column 'OpenAI Whisper V3 (1.5B)', row 'Fleurs EN' = 6.23 (CER/WER %, note 1: 'Values represent CER/WER'). Printed table cell, hand-read; Z.ai's own runs (only the Seed-ASR column is marked as taken from the FunAudio-ASR report and is not used).  Whisper V3 = openai/whisper-large-v3 as run by Z.ai; decoding not stated.",
    "source_id": "speech-glm-asr-readme-apache"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.706,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'IndexTTS2', column test-en SIM-o = 0.706. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0233,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'IndexTTS2', column test-en WER = 2.33. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.764,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'IndexTTS2', column test-zh SIM-o = 0.764. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "indextts-2",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0105,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'IndexTTS2', column test-zh WER/CER = 1.05. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "zipvoice",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.697,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'ZipVoice', column test-en SIM-o = 0.697. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "zipvoice",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.017,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'ZipVoice', column test-en WER = 1.70. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "zipvoice",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.751,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'ZipVoice', column test-zh SIM-o = 0.751. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "zipvoice",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.014,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'ZipVoice', column test-zh WER/CER = 1.40. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.713,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'MaskGCT', column test-en SIM-o = 0.713. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0288,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'MaskGCT', column test-en WER = 2.88. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.773,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'MaskGCT', column test-zh SIM-o = 0.773. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "maskgct",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.024,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "independent",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'MaskGCT', column test-zh WER/CER = 2.40. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.741,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "lab",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'OmniVoice', column test-en SIM-o = 0.741. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.016,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "lab",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'OmniVoice', column test-en WER = 1.60. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.777,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "lab",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'OmniVoice', column test-zh SIM-o = 0.777. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0084,
    "source": "https://arxiv.org/html/2604.00688v3",
    "by": "lab",
    "date": "2026-04-01",
    "note": "OmniVoice report (k2-fsa, arXiv:2604.00688v3, CC BY 4.0) Table 1 bottom (Seed-TTS test sets), row 'OmniVoice', column test-zh WER/CER = 0.84. 'Baseline results are obtained using official checkpoints' (authors' own runs). §3.5: SIM-o with a WavLM-based ECAPA-TDNN; WER with Whisper-large-v3 (en) and Paraformer-zh (zh, CER reported as WER).",
    "source_id": "speech-omnivoice-report-ccby"
  },
  {
    "model": "gpt-4-1",
    "bench": "bfcl-v3",
    "score": 0.6889,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4.1-2025-04-14 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 68.89%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://openai.com/index/gpt-4-1/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-14b",
    "bench": "bfcl-v3",
    "score": 0.6801,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-14B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 68.01%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/Qwen/Qwen3-14B. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-8b",
    "bench": "bfcl-v3",
    "score": 0.6634,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-8B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 66.34%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/Qwen/Qwen3-8B. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "bfcl-v3",
    "score": 0.6518,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4.1-mini-2025-04-14 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 65.18%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://openai.com/index/gpt-4-1/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "command-a",
    "bench": "bfcl-v3",
    "score": 0.6437,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Command A (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 64.37%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://cohere.com/blog/command-a. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-pro",
    "bench": "bfcl-v3",
    "score": 0.6082,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Amazon-Nova-Pro-v1:0 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 60.82%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://aws.amazon.com/cn/ai/generative-ai/nova/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-27b",
    "bench": "bfcl-v3",
    "score": 0.5911,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemma-3-27b-it (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 59.11%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://blog.google/technology/developers/gemma-3/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-large-2",
    "bench": "bfcl-v3",
    "score": 0.5803,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'mistral-large-2411 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 58.03%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://docs.mistral.ai/guides/model-selection/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "bfcl-v3",
    "score": 0.5668,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.1-Nemotron-Ultra-253B-v1 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.68%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bfcl-v3",
    "score": 0.547,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-1.7B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 54.70%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/Qwen/Qwen3-1.7B. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "bfcl-v3",
    "score": 0.5298,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4.1-nano-2025-04-14 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.98%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://openai.com/index/gpt-4-1/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-4-maverick",
    "bench": "bfcl-v3",
    "score": 0.5286,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.86%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bfcl-v3",
    "score": 0.5224,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.3-70B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.24%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://llama.meta.com/llama3. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-micro",
    "bench": "bfcl-v3",
    "score": 0.5183,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Amazon-Nova-Micro-v1:0 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 51.83%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://aws.amazon.com/cn/ai/generative-ai/nova/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "bfcl-v3",
    "score": 0.517,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Command R7B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 51.70%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://cohere.com/blog/command-r7b. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "bfcl-v3",
    "score": 0.5154,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Ministral-8B-Instruct-2410 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 51.54%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/mistralai/Ministral-8B-Instruct-2410. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "bfcl-v3",
    "score": 0.5148,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Open-Mistral-Nemo-2407 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 51.48%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://mistral.ai/news/mistral-nemo/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-12b",
    "bench": "bfcl-v3",
    "score": 0.5063,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemma-3-12b-it (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 50.63%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://blog.google/technology/developers/gemma-3/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bfcl-v3",
    "score": 0.4957,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.1-8B-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 49.57%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://llama.meta.com/llama3. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "bfcl-v3",
    "score": 0.4733,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-0.6B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 47.33%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/Qwen/Qwen3-0.6B. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "phi-4",
    "bench": "bfcl-v3",
    "score": 0.4697,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Phi-4 (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 46.97%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/microsoft/phi-4. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "bfcl-v3",
    "score": 0.4586,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.2-3B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 45.86%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://llama.meta.com/llama3. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-4-scout",
    "bench": "bfcl-v3",
    "score": 0.4541,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-4-Scout-17B-16E-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 45.41%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "falcon3-10b",
    "bench": "bfcl-v3",
    "score": 0.4513,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Falcon3-10B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 45.13%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/tiiuae/Falcon3-10B-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "falcon3-7b",
    "bench": "bfcl-v3",
    "score": 0.4359,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Falcon3-7B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 43.59%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/tiiuae/Falcon3-7B-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-4b",
    "bench": "bfcl-v3",
    "score": 0.3963,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemma-3-4b-it (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 39.63%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://blog.google/technology/developers/gemma-3/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "dbrx-instruct",
    "bench": "bfcl-v3",
    "score": 0.3921,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'DBRX-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 39.21%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://www.databricks.com/blog/introducing-dbrx-new-state-art-open-llm. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "falcon3-3b",
    "bench": "bfcl-v3",
    "score": 0.3247,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Falcon3-3B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 32.47%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/tiiuae/Falcon3-3B-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bfcl-v3",
    "score": 0.2195,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.2-1B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 21.95%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://llama.meta.com/llama3. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "falcon3-1b",
    "bench": "bfcl-v3",
    "score": 0.1967,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Falcon3-1B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 19.67%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://huggingface.co/tiiuae/Falcon3-1B-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemma-3-1b",
    "bench": "bfcl-v3",
    "score": 0.1632,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemma-3-1b-it (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 16.32%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt, as for the existing V4 rows; no component scores. Model mapping reuses the reviewed V4 entry name, Model Link https://blog.google/technology/developers/gemma-3/. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "aider-code-editing",
    "score": 0.684,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-01",
    "note": "Published entry 'claude-3-opus-20240229'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 68.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.30.2-dev; evaluator commit=f4b1797-dirty, f4b1797; command=aider --opus; run=2024-05-01-20-05-59--direct-opus-filenames-outside-fence. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "aider-code-editing",
    "score": 0.549,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-03-06",
    "note": "Published entry 'claude-3-sonnet-20240229'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 54.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.25.1-dev; evaluator commit=a5f8076-dirty; command=aider --sonnet; run=2024-03-06-16-42-00--claude3-sonnet-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "aider-code-editing",
    "score": 0.504,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-08",
    "note": "Published entry 'gpt-3.5-turbo-0125'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 50.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.33.1-dev; evaluator commit=1d55f74; command=aider -3; run=2024-05-08-20-59-15--may-gpt-3.5-turbo-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "aider-code-editing",
    "score": 0.579,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2023-11-06",
    "note": "Published entry 'gpt-3.5-turbo-0301'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 57.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.16.4-dev; evaluator commit=44388db-dirty; command=aider --model gpt-3.5-turbo-0301; run=2023-11-06-21-23-59--gpt-3.5-turbo-0301. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "aider-code-editing",
    "score": 0.504,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2023-11-07",
    "note": "Published entry 'gpt-3.5-turbo-0613'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 50.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.16.4-dev; evaluator commit=93aa497-dirty; command=aider --model gpt-3.5-turbo-0613; run=2023-11-07-02-41-07--gpt-3.5-turbo-0613. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "aider-code-editing",
    "score": 0.662,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-01-25",
    "note": "Published entry 'gpt-4-0125-preview'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 66.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=udiff; Aider version=0.22.1-dev; evaluator commit=edcf9b1; command=aider --model gpt-4-0125-preview; run=2024-01-25-23-37-15--jan-exercism-gpt-4-0125-preview-udiff. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4-0314",
    "bench": "aider-code-editing",
    "score": 0.662,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-04",
    "note": "Published entry 'gpt-4-0314'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 66.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.31.2-dev; evaluator commit=0d43468; command=aider --model gpt-4-0314; run=2024-05-04-15-07-30--redo-gpt-4-0314-diff-reminder-rules. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4-0613",
    "bench": "aider-code-editing",
    "score": 0.677,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2023-12-16",
    "note": "Published entry 'gpt-4-0613'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 67.7 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.18.2-dev; evaluator commit=3aa17c4; command=aider -4; run=2023-12-16-21-24-28--editblock-gpt-4-0613-actual-main. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "aider-code-editing",
    "score": 0.654,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-08",
    "note": "Published entry 'gpt-4-1106-preview'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 65.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=udiff; Aider version=0.33.1-dev; evaluator commit=87664dc; command=aider --model gpt-4-1106-preview; run=2024-05-08-21-16-03--may-gpt-4-1106-preview-udiff. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "aider-code-editing",
    "score": 0.639,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-01",
    "note": "Published entry 'gpt-4-turbo-2024-04-09 (udiff)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 63.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=udiff; Aider version=0.30.2-dev; evaluator commit=e610e5b-dirty; command=aider --gpt-4-turbo; run=2024-05-01-02-09-20--gpt-4-turbo-examples. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "aider-code-editing",
    "score": 0.316,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-06",
    "note": "Published entry 'command-r-plus'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 31.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.31.2-dev; evaluator commit=fc3a43e-dirty; command=aider --model command-r-plus; run=2024-05-06-18-31-08--command-r-plus-whole-final. Run 2024-05-06, before the 08-2024 release, so API name command-r-plus was the 04-2024 model. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "aider-code-editing",
    "score": 0.376,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Published entry 'qwen1.5-110b-chat'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 37.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.31.2-dev; evaluator commit=70b1c0c; command=aider --model together_ai/qwen/qwen1.5-110b-chat; run=2024-05-07-20-32-37--qwen1.5-110b-chat-whole. Served by Together AI (together_ai/qwen/qwen1.5-110b-chat). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "aider-code-editing",
    "score": 0.444,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Published entry 'WizardLM-2 8x22B'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 44.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.31.2-dev; evaluator commit=8e272bf, bbe8639; command=aider --model openrouter/microsoft/wizardlm-2-8x22b; run=2024-05-07-20-57-04--wizardlm-2-8x22b-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "aider-code-editing",
    "score": 0.729,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-05-13",
    "note": "Published entry 'gpt-4o-2024-05-13'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 72.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.34.1-dev; evaluator commit=b6cd852; command=aider; run=2024-05-13-17-39-05--gpt-4o-diff. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-72b",
    "bench": "aider-code-editing",
    "score": 0.556,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-06-08",
    "note": "Published entry 'Qwen2 72B Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 55.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.37.1-dev; evaluator commit=02c7335-dirty, 1a97498-dirty; command=aider --model together_ai/qwen/Qwen2-72B-Instruct; run=2024-06-08-22-37-55--qwen2-72b-instruct-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "aider-code-editing",
    "score": 0.511,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-06-09",
    "note": "Published entry 'codestral-2405'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 51.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.37.1-dev; evaluator commit=effc88a; command=aider --model mistral/codestral-2405; run=2024-06-09-03-28-21--codestral-whole. Mistral docs model page https://docs.mistral.ai/models/codestral-24-05 (observed 2026-10-07) gives API name codestral-2405 for Codestral v24.05, 22B, with weights link huggingface.co/mistralai/Codestral-22B-v0.1. Quantised Ollama/GGUF entries of the same model not used. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "aider-code-editing",
    "score": 0.774,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-04",
    "note": "Published entry 'claude-3.5-sonnet-20240620'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 77.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.42.1-dev; evaluator commit=35f21b5; command=aider --model claude-3.5-sonnet-20240620; run=2024-07-04-14-32-08--claude-3.5-sonnet-diff-continue. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "aider-code-editing",
    "score": 0.474,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-01",
    "note": "Published entry 'claude-3-haiku-20240307'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 47.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.41.1-dev; evaluator commit=75f506d; command=aider --model claude-3-haiku-20240307; run=2024-07-01-21-41-48--haiku-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "aider-code-editing",
    "score": 0.556,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-18",
    "note": "Published entry 'gpt-4o-mini'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 55.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.44.1-dev; evaluator commit=d31eef3-dirty; command=aider --model gpt-4o-mini; run=2024-07-18-18-57-46--gpt-4o-mini-whole. Run 2024-07-18; gpt-4o-mini-2024-07-18 was the only snapshot. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "aider-code-editing",
    "score": 0.662,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-24",
    "note": "Published entry 'llama-3.1-405b-instruct (whole)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 66.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.45.2-dev; evaluator commit=a362dea-dirty; command=aider --model openrouter/meta-llama/llama-3.1-405b-instruct; run=2024-07-24-06-30-29--llama-405b-whole. Higher of the two full runs (whole 66.2; diff 63.9). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "mistral-large-2407",
    "bench": "aider-code-editing",
    "score": 0.602,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-24",
    "note": "Published entry 'Mistral Large 2 (2407)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 60.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.45.2-dev; evaluator commit=859a13e; command=aider --model mistral/mistral-large-2407; run=2024-07-24-19-08-47--mistral-large-2407-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "aider-code-editing",
    "score": 0.376,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-25",
    "note": "Published entry 'llama-3.1-8b-instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 37.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.45.2-dev; evaluator commit=ffcced8; command=aider --model fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct; run=2024-07-25-08-12-27--fireworks-llama-8b-whole. Served by Fireworks. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "aider-code-editing",
    "score": 0.586,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-25",
    "note": "Published entry 'llama-3.1-70b-instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 58.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.45.2-dev; evaluator commit=ffcced8; command=aider --model fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct; run=2024-07-25-08-07-45--fireworks-llama-70b-whole. Served by Fireworks. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "aider-code-editing",
    "score": 0.714,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-08-06",
    "note": "Published entry 'gpt-4o-2024-08-06'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 71.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.48.1-dev; evaluator commit=ed9ed89; command=aider --model openai/gpt-4o-2024-08-06; run=2024-08-06-18-28-39--gpt-4o-2024-08-06-diff-again. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "aider-code-editing",
    "score": 0.541,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-04",
    "note": "Published entry 'Yi Coder 9B Chat'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 54.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.54.13.dev; evaluator commit=c4e4967; command=aider --model openai/hf:01-ai/Yi-Coder-9B-Chat --openai-api-base https://glhf.chat/api/openai/v1; run=2024-09-04-16-08-09--yi-coder-9b-whole. HF checkpoint 01-ai/Yi-Coder-9B-Chat served by glhf.chat (not the quantised Ollama runs). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "aider-code-editing",
    "score": 0.722,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-05",
    "note": "Published entry 'DeepSeek V2.5'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 72.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.55.1.dev; evaluator commit=1279c86; command=aider --deepseek; run=2024-09-05-14-50-11--deepseek-sep5-no-shell. Run 2024-09-05 with deepseek-chat, labelled DeepSeek V2.5. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "aider-code-editing",
    "score": 0.383,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-11",
    "note": "Published entry 'Command R+ (08-24)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 38.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.56.1.dev; evaluator commit=b43ed20; command=aider --model command-r-plus-08-2024; run=2024-09-11-15-42-17--command-r-plus-08-2024-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "command-r-08-2024",
    "bench": "aider-code-editing",
    "score": 0.383,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-11",
    "note": "Published entry 'Command R (08-24)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 38.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.56.1.dev; evaluator commit=b43ed20-dirty; command=aider --model command-r-08-2024; run=2024-09-11-15-47-02--command-r-08-2024-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "o1-mini",
    "bench": "aider-code-editing",
    "score": 0.707,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-12",
    "note": "Published entry 'o1-mini (whole)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 70.7 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.56.1.dev; evaluator commit=36fa773-dirty, 291b456; command=aider --model o1-mini; run=2024-09-12-19-57-35--o1-mini-whole. Full 133-case run (the 36-case diff entry not used). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "o1-preview",
    "bench": "aider-code-editing",
    "score": 0.797,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-21",
    "note": "Published entry 'o1-preview'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 79.7 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.56.1.dev; evaluator commit=5493654-dirty; command=aider --model o1-preview; run=2024-09-21-16-45-11--o1-preview-flex-sr-markers. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "aider-code-editing",
    "score": 0.654,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-20",
    "note": "Published entry 'qwen-2.5-72b-instruct (bf16)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 65.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.56.1.dev; evaluator commit=5139594; command=aider --model openrouter/qwen/qwen-2.5-72b-instruct; run=2024-09-20-20-20-19--qwen-2.5-72b-instruct-diff. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "aider-code-editing",
    "score": 0.654,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-24",
    "note": "Published entry 'gemini-1.5-pro-002'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 65.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff-fenced; Aider version=0.57.2.dev; evaluator commit=6b5fe9b, 3edcd71; command=aider --model gemini/gemini-1.5-pro-002; run=2024-09-24-16-26-45--gemini-1.5-pro-002-diff-fenced. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "aider-code-editing",
    "score": 0.511,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-24",
    "note": "Published entry 'gemini-1.5-flash-002 (0924)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 51.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.57.2.dev; evaluator commit=3edcd71; command=aider --model gemini/gemini-1.5-flash-002; run=2024-09-24-16-33-23--gemini-1.5-flash-002-whole. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "aider-code-editing",
    "score": 0.301,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-30",
    "note": "Published entry 'ollama/hermes3:8b-llama3.1-fp16'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 30.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.58.1.dev; evaluator commit=c5ba4f7; command=aider --model ollama/hermes3:8b-llama3.1-fp16; run=2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2. Ollama hermes3:8b-llama3.1-fp16 tag: unquantised fp16 build of NousResearch/Hermes-3-Llama-3.1-8B (the default quantised hermes3 tag entry not used). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "aider-code-editing",
    "score": 0.263,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-30",
    "note": "Published entry 'ollama/llama3.2:3b-instruct-fp16'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 26.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.58.1.dev; evaluator commit=3f12290; command=aider --model ollama/llama3.2:3b-instruct-fp16; run=2024-09-30-19-35-40--llama3.2-3b-instruct-fp16-whole-1. Ollama llama3.2:3b-instruct-fp16 tag: unquantised fp16 build. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "aider-code-editing",
    "score": 0.549,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-16",
    "note": "Published entry 'Llama-3.1-Nemotron-70B-Instruct-HF'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 54.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=6bb9b25-dirty; command=(via glhf.chat); run=2024-10-16-15-55-37--nemotron-glhf-whole3. HF checkpoint served by glhf.chat. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "aider-code-editing",
    "score": 0.842,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-22",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 84.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.59.2.dev; evaluator commit=3b14eb9; command=aider --model anthropic/claude-3-5-sonnet-20241022; run=2024-10-22-17-45-28--sonnet-1022-diff-fixed-model-settings. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "aider-code-editing",
    "score": 0.752,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-04",
    "note": "Published entry 'claude-3-5-haiku-20241022'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 75.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.61.1.dev; evaluator commit=03bbdb0-dirty; command=aider --model anthropic/claude-3-5-haiku-20241022; run=2024-11-04-19-19-32--haiku35-diff-ex-as-sys-false. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "aider-code-editing",
    "score": 0.579,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-16",
    "note": "Published entry 'Qwen2.5-Coder-7B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 57.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=92fe979-dirty; command=aider --model openai/Qwen2.5-Coder-7B-Instruct; run=2024-10-16-16-20-59--Qwen2.5-Coder-7B-Instruct. OpenAI-compatible endpoint serving Qwen2.5-Coder-7B-Instruct (not the quantised Ollama/GGUF entries). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "aider-code-editing",
    "score": 0.692,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-29",
    "note": "Published entry 'Qwen2.5-Coder-14B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 69.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=58bd375; command=aider --model openai/Qwen2.5-Coder-14B-Instruct; run=2024-10-29-11-53-39--Qwen2.5-Coder-14B-Instruct. OpenAI-compatible endpoint serving Qwen2.5-Coder-14B-Instruct (not the quantised Ollama entry). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "aider-code-editing",
    "score": 0.714,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-09",
    "note": "Published entry 'Qwen2.5-Coder-32B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 71.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.59.2.dev; evaluator commit=ec9982a; command=aider --model openai/hf:Qwen/Qwen2.5-Coder-32B-Instruct --openai-api-base https://glhf.chat/api/openai/v1; run=2024-11-09-11-09-15--Qwen2.5-Coder-32B-Instruct. HF checkpoint served by glhf.chat; higher of the two full API runs (glhf diff 71.4; OpenRouter diff 65.4); quantised Ollama entry not used. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "mistral-large-2",
    "bench": "aider-code-editing",
    "score": 0.654,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-20",
    "note": "Published entry 'Mistral Large (2411)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 65.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.63.3.dev; evaluator commit=dba844c; command=aider --model mistral/mistral-large-latest; run=2024-11-20-14-57-11--mistral-2411-direct-diff. Run 2024-11-20 with mistral-large-latest, labelled Mistral Large (2411). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "aider-code-editing",
    "score": 0.714,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-20",
    "note": "Published entry 'gpt-4o-2024-11-20'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 71.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.63.3.dev; evaluator commit=2ac0776-dirty; command=aider --model openai/gpt-4o-2024-11-20; run=2024-11-20-19-28-30--gpt-4o-2024-11-20. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "nova-pro",
    "bench": "aider-code-editing",
    "score": 0.541,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-04",
    "note": "Published entry 'Nova Pro'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 54.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.66.1.dev; evaluator commit=699e283; command=aider --model bedrock/us.amazon.nova-pro-v1:0; run=2024-12-04-13-53-03--nova-whole. Amazon Bedrock us.amazon.nova-pro-v1:0. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "aider-code-editing",
    "score": 0.594,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-06",
    "note": "Published entry 'llama-3.3-70b-instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 59.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.67.1.dev; evaluator commit=53e0d67; command=aider --model openrouter/meta-llama/llama-3.3-70b-instruct; run=2024-12-06-18-27-47--llama33-diff. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "aider-code-editing",
    "score": 0.722,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-10",
    "note": "Published entry 'DeepSeek-V2.5-1210'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 72.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.67.1.dev; evaluator commit=16332b2; command=aider --model deepseek/deepseek-chat; run=2024-12-10-14-45-21--deepseek-1210-diff. Run 2024-12-10 with deepseek-chat, labelled DeepSeek-V2.5-1210 (released that day). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "yi-lightning",
    "bench": "aider-code-editing",
    "score": 0.654,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-11",
    "note": "Published entry 'yi-lightning'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 65.4 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.67.1.dev; evaluator commit=e909a3d-dirty; command=aider --model openai/yi-lightning; run=2024-12-11-00-37-08--yi-test. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "aider-code-editing",
    "score": 0.842,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-18",
    "note": "Published entry 'o1'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 84.2 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.69.2.dev; evaluator commit=074c636-dirty; command=aider --model openrouter/openai/o1; run=2024-12-18-01-50-08--o1. Run 2024-12-18; o1-2024-12-17 was the only o1 snapshot. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2852,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '6.17 ± 0.99 / 28.52 ± 0.69' (MCC / accuracy); entry 'ibm-granite/granite-3b-code-instruct-2k', instruction-tuned (📝), default setting; EuroEval version 15.2.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5152,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '36.29 ± 1.97 / 51.52 ± 1.73' (MCC / accuracy); entry 'ibm-granite/granite-34b-code-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0. Hugging Face redirects the entry repo ibm-granite/granite-34b-code-instruct to ibm-granite/granite-34b-code-instruct-8k (same repository _id, renamed), the catalogue checkpoint.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3413,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '16.24 ± 1.41 / 34.13 ± 1.00' (MCC / accuracy); entry 'ibm-granite/granite-34b-code-instruct', instruction-tuned (📝), default setting; EuroEval version 15.8.2.dev0. Hugging Face redirects the entry repo ibm-granite/granite-34b-code-instruct to ibm-granite/granite-34b-code-instruct-8k (same repository _id, renamed), the catalogue checkpoint.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.4541,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '25.67 ± 1.74 / 45.41 ± 1.46' (MCC / accuracy); entry 'ibm-granite/granite-20b-code-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0. Hugging Face redirects the entry repo ibm-granite/granite-20b-code-instruct to ibm-granite/granite-20b-code-instruct-8k (same repository _id, renamed), the catalogue checkpoint.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3308,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '12.06 ± 1.18 / 33.08 ± 0.95' (MCC / accuracy); entry 'ibm-granite/granite-20b-code-instruct', instruction-tuned (📝), default setting; EuroEval version 15.9.2.dev0. Hugging Face redirects the entry repo ibm-granite/granite-20b-code-instruct to ibm-granite/granite-20b-code-instruct-8k (same repository _id, renamed), the catalogue checkpoint.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "falcon-40b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6027,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/falcon-40b' (source=hub, precision=bfloat16, type=pretrained); published average 60.27 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 4a70170c21. Pretrained base model, as in the catalogue entry (not falcon-40b-instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "olmo-7b-twin-2t",
    "bench": "open-pt-llm-avg9",
    "score": 0.4503,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-17",
    "note": "Open PT-LLM Leaderboard row 'allenai/OLMo-7B-Twin-2T' (source=hub, precision=float16, type=pretrained); published average 45.03 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a77e93a129. Original allenai/OLMo-7B-Twin-2T repository (hf_olmo format); the catalogue links AI2's Transformers-format conversion OLMo-7B-Twin-2T-hf of the same checkpoint.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.0196,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'openai-community/openai-gpt' (source=hub, precision=float16, type=pretrained); published average 1.96 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 61cf0c79fa. All nine task scores are at or near zero as published (e.g. ENEM 0.0, BLUEX 0.0, OAB 0.0); recorded as published.",
    "source_id": "open-pt-llm-leaderboard",
    "invalid": "Already fixed by chance 0.3264 (c7b3953). 1.96% average vs a 32.64% random baseline; GPT-1's 512-token context cannot hold 3- to 25-shot prompts."
  },
  {
    "model": "phi-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.2754,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'microsoft/phi-1' (source=hub, precision=float16, type=pretrained); published average 27.54 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a1a59319a7. Base Phi-1 (1.3B code model), as in the catalogue entry.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "chatglm3-6b",
    "bench": "open-pt-llm-avg9",
    "score": 0.56,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-19",
    "note": "Open PT-LLM Leaderboard row 'THUDM/chatglm3-6b' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 56.0 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 103caa4002. Repository since renamed to zai-org/chatglm3-6b; weight files at the evaluated commit 103caa4002 are identical to the catalogue-pinned revision e9e0406 (all 15 LFS sha256 match).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "chatglm2-6b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4717,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-05",
    "note": "Open PT-LLM Leaderboard row 'THUDM/chatglm2-6b' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 47.17 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 7fabe56db9. Repository since renamed to zai-org/chatglm2-6b; weight files at the evaluated commit 7fabe56db9 are identical to the catalogue-pinned revision d2e2d91 (all 8 LFS sha256 match).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "flan-t5-small",
    "bench": "mgsm",
    "score": 0.002,
    "source": "https://arxiv.org/html/2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Small', column 'MGSM CoT' = 0.2 (few-shot chain-of-thought prompting, exact-match accuracy, unweighted average over the 10 MGSM languages per the caption). Google's own evaluation of its released instruction-finetuned checkpoint. Same table and run as the existing MMLU Direct / BBH Direct rows; MMLU CoT, BBH CoT and TyDiQA columns not added. CC BY 4.0 (arXiv licence).",
    "source_id": "paper-2210.11416v5"
  },
  {
    "model": "flan-t5-base",
    "bench": "mgsm",
    "score": 0.004,
    "source": "https://arxiv.org/html/2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Base', column 'MGSM CoT' = 0.4 (few-shot chain-of-thought prompting, exact-match accuracy, unweighted average over the 10 MGSM languages per the caption). Google's own evaluation of its released instruction-finetuned checkpoint. Same table and run as the existing MMLU Direct / BBH Direct rows; MMLU CoT, BBH CoT and TyDiQA columns not added. CC BY 4.0 (arXiv licence).",
    "source_id": "paper-2210.11416v5"
  },
  {
    "model": "flan-t5-large",
    "bench": "mgsm",
    "score": 0.007,
    "source": "https://arxiv.org/html/2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-Large', column 'MGSM CoT' = 0.7 (few-shot chain-of-thought prompting, exact-match accuracy, unweighted average over the 10 MGSM languages per the caption). Google's own evaluation of its released instruction-finetuned checkpoint. Same table and run as the existing MMLU Direct / BBH Direct rows; MMLU CoT, BBH CoT and TyDiQA columns not added. CC BY 4.0 (arXiv licence).",
    "source_id": "paper-2210.11416v5"
  },
  {
    "model": "flan-t5-xl",
    "bench": "mgsm",
    "score": 0.019,
    "source": "https://arxiv.org/html/2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XL', column 'MGSM CoT' = 1.9 (few-shot chain-of-thought prompting, exact-match accuracy, unweighted average over the 10 MGSM languages per the caption). Google's own evaluation of its released instruction-finetuned checkpoint. Same table and run as the existing MMLU Direct / BBH Direct rows; MMLU CoT, BBH CoT and TyDiQA columns not added. CC BY 4.0 (arXiv licence).",
    "source_id": "paper-2210.11416v5"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "mgsm",
    "score": 0.049,
    "source": "https://arxiv.org/html/2210.11416v5",
    "by": "lab",
    "date": "2022-12-06",
    "note": "Flan paper (arXiv:2210.11416v5) Table 5, row 'Flan-T5-XXL', column 'MGSM CoT' = 4.9 (few-shot chain-of-thought prompting, exact-match accuracy, unweighted average over the 10 MGSM languages per the caption). Google's own evaluation of its released instruction-finetuned checkpoint. Same table and run as the existing MMLU Direct / BBH Direct rows; MMLU CoT, BBH CoT and TyDiQA columns not added. CC BY 4.0 (arXiv licence).",
    "source_id": "paper-2210.11416v5"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "humaneval",
    "score": 0.921,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen2.5-Coder-32B-Instruct' (link https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct; chat/instruction prompt); pass@1 humaneval = 92.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "humaneval-plus",
    "score": 0.872,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen2.5-Coder-32B-Instruct' (link https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 87.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "mbpp-plus",
    "score": 0.77,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen2.5-Coder-32B-Instruct' (link https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct; chat/instruction prompt); pass@1 mbpp+ = 77.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "humaneval",
    "score": 0.835,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B-Chat' (link https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat; chat/instruction prompt); pass@1 humaneval = 83.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "humaneval-plus",
    "score": 0.787,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B-Chat' (link https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat; chat/instruction prompt); pass@1 humaneval+ = 78.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "mbpp-plus",
    "score": 0.69,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B-Chat' (link https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat; chat/instruction prompt); pass@1 mbpp+ = 69.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "humaneval",
    "score": 0.683,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen1.5-72B-Chat' (link https://huggingface.co/Qwen/Qwen1.5-72B-Chat; chat/instruction prompt); pass@1 humaneval = 68.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "humaneval-plus",
    "score": 0.591,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen1.5-72B-Chat' (link https://huggingface.co/Qwen/Qwen1.5-72B-Chat; chat/instruction prompt); pass@1 humaneval+ = 59.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "mbpp-plus",
    "score": 0.616,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Qwen1.5-72B-Chat' (link https://huggingface.co/Qwen/Qwen1.5-72B-Chat; chat/instruction prompt); pass@1 mbpp+ = 61.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "humaneval",
    "score": 0.854,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-V2-Instruct' (link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct; chat/instruction prompt); pass@1 humaneval = 85.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "humaneval-plus",
    "score": 0.823,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-V2-Instruct' (link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct; chat/instruction prompt); pass@1 humaneval+ = 82.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "mbpp-plus",
    "score": 0.751,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-V2-Instruct' (link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct; chat/instruction prompt); pass@1 mbpp+ = 75.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base; direct code completion); pass@1 humaneval = 28.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "humaneval-plus",
    "score": 0.256,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base; direct code completion); pass@1 humaneval+ = 25.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "mbpp-plus",
    "score": 0.479,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base; direct code completion); pass@1 mbpp+ = 47.9 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "humaneval",
    "score": 0.659,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct; chat/instruction prompt); pass@1 humaneval = 65.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "humaneval-plus",
    "score": 0.604,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct; chat/instruction prompt); pass@1 humaneval+ = 60.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "mbpp-plus",
    "score": 0.548,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-1.3B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct; chat/instruction prompt); pass@1 mbpp+ = 54.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "humaneval",
    "score": 0.512,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-33B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-33b-base; direct code completion); pass@1 humaneval = 51.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "humaneval-plus",
    "score": 0.445,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-33B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-33b-base; direct code completion); pass@1 humaneval+ = 44.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "humaneval",
    "score": 0.811,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-33B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct; chat/instruction prompt); pass@1 humaneval = 81.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "humaneval-plus",
    "score": 0.75,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-33B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct; chat/instruction prompt); pass@1 humaneval+ = 75.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "mbpp-plus",
    "score": 0.701,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-33B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct; chat/instruction prompt); pass@1 mbpp+ = 70.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "humaneval",
    "score": 0.476,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base; direct code completion); pass@1 humaneval = 47.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "humaneval-plus",
    "score": 0.396,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base; direct code completion); pass@1 humaneval+ = 39.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "mbpp-plus",
    "score": 0.587,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-base' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base; direct code completion); pass@1 mbpp+ = 58.7 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "humaneval",
    "score": 0.744,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct; chat/instruction prompt); pass@1 humaneval = 74.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "humaneval-plus",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct; chat/instruction prompt); pass@1 humaneval+ = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "mbpp-plus",
    "score": 0.656,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-6.7B-instruct' (link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct; chat/instruction prompt); pass@1 mbpp+ = 65.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "humaneval",
    "score": 0.756,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-7B-instruct-v1.5' (link https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5; chat/instruction prompt); pass@1 humaneval = 75.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "humaneval-plus",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-7B-instruct-v1.5' (link https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5; chat/instruction prompt); pass@1 humaneval+ = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "mbpp-plus",
    "score": 0.622,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'DeepSeek-Coder-7B-instruct-v1.5' (link https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5; chat/instruction prompt); pass@1 mbpp+ = 62.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-70b",
    "bench": "humaneval",
    "score": 0.774,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-70B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct; chat/instruction prompt); pass@1 humaneval = 77.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-70b",
    "bench": "humaneval-plus",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-70B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-70b",
    "bench": "mbpp-plus",
    "score": 0.69,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-70B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct; chat/instruction prompt); pass@1 mbpp+ = 69.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "humaneval",
    "score": 0.695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3.1-8B-instruct' (link https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct; chat/instruction prompt); pass@1 humaneval = 69.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "humaneval-plus",
    "score": 0.628,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3.1-8B-instruct' (link https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 62.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mbpp-plus",
    "score": 0.556,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3.1-8B-instruct' (link https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct; chat/instruction prompt); pass@1 mbpp+ = 55.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b",
    "bench": "humaneval",
    "score": 0.616,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct; chat/instruction prompt); pass@1 humaneval = 61.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b",
    "bench": "humaneval-plus",
    "score": 0.567,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 56.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b",
    "bench": "mbpp-plus",
    "score": 0.548,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-instruct' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct; chat/instruction prompt); pass@1 mbpp+ = 54.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-DS-6.7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B; chat/instruction prompt); pass@1 humaneval = 76.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "humaneval-plus",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-DS-6.7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B; chat/instruction prompt); pass@1 humaneval+ = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "mbpp-plus",
    "score": 0.69,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-DS-6.7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B; chat/instruction prompt); pass@1 mbpp+ = 69.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "humaneval",
    "score": 0.799,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Codestral-22B-v0.1' (link https://huggingface.co/mistralai/Codestral-22B-v0.1; chat/instruction prompt); pass@1 humaneval = 79.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "humaneval-plus",
    "score": 0.738,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Codestral-22B-v0.1' (link https://huggingface.co/mistralai/Codestral-22B-v0.1; chat/instruction prompt); pass@1 humaneval+ = 73.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "mbpp-plus",
    "score": 0.619,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Codestral-22B-v0.1' (link https://huggingface.co/mistralai/Codestral-22B-v0.1; chat/instruction prompt); pass@1 mbpp+ = 61.9 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humaneval",
    "score": 0.421,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B-Instruct-v0.2' (link https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2; chat/instruction prompt); pass@1 humaneval = 42.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humaneval-plus",
    "score": 0.36,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B-Instruct-v0.2' (link https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2; chat/instruction prompt); pass@1 humaneval+ = 36.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "mbpp-plus",
    "score": 0.37,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B-Instruct-v0.2' (link https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2; chat/instruction prompt); pass@1 mbpp+ = 37.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "humaneval",
    "score": 0.762,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x22B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1; chat/instruction prompt); pass@1 humaneval = 76.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "humaneval-plus",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x22B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1; chat/instruction prompt); pass@1 humaneval+ = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "mbpp-plus",
    "score": 0.643,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x22B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1; chat/instruction prompt); pass@1 mbpp+ = 64.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humaneval",
    "score": 0.451,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x7B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1; chat/instruction prompt); pass@1 humaneval = 45.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humaneval-plus",
    "score": 0.396,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x7B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1; chat/instruction prompt); pass@1 humaneval+ = 39.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "mbpp-plus",
    "score": 0.497,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mixtral-8x7B-Instruct-v0.1' (link https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1; chat/instruction prompt); pass@1 mbpp+ = 49.7 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "humaneval",
    "score": 0.726,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenChat-3.5-7B-0106' (link https://huggingface.co/openchat/openchat-3.5-0106; chat/instruction prompt); pass@1 humaneval = 72.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "humaneval-plus",
    "score": 0.677,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenChat-3.5-7B-0106' (link https://huggingface.co/openchat/openchat-3.5-0106; chat/instruction prompt); pass@1 humaneval+ = 67.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "mbpp-plus",
    "score": 0.545,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenChat-3.5-7B-0106' (link https://huggingface.co/openchat/openchat-3.5-0106; chat/instruction prompt); pass@1 mbpp+ = 54.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "humaneval",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Phind-CodeLlama-34B-v2' (link https://huggingface.co/Phind/Phind-CodeLlama-34B-v2; direct code completion); pass@1 humaneval = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phind-codellama-34b-v2",
    "bench": "humaneval-plus",
    "score": 0.671,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Phind-CodeLlama-34B-v2' (link https://huggingface.co/Phind/Phind-CodeLlama-34B-v2; direct code completion); pass@1 humaneval+ = 67.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "humaneval",
    "score": 0.433,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'SOLAR-10.7B-Instruct-v1.0' (link https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0; chat/instruction prompt); pass@1 humaneval = 43.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "humaneval-plus",
    "score": 0.372,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'SOLAR-10.7B-Instruct-v1.0' (link https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0; chat/instruction prompt); pass@1 humaneval+ = 37.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "mbpp-plus",
    "score": 0.362,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'SOLAR-10.7B-Instruct-v1.0' (link https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0; chat/instruction prompt); pass@1 mbpp+ = 36.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "humaneval",
    "score": 0.024,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StableLM-7B' (link https://huggingface.co/stabilityai/stablelm-base-alpha-7b; direct code completion); pass@1 humaneval = 2.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "humaneval-plus",
    "score": 0.024,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StableLM-7B' (link https://huggingface.co/stabilityai/stablelm-base-alpha-7b; direct code completion); pass@1 humaneval+ = 2.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder",
    "bench": "humaneval",
    "score": 0.341,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder-15B' (link https://huggingface.co/bigcode/starcoder; direct code completion); pass@1 humaneval = 34.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder",
    "bench": "humaneval-plus",
    "score": 0.293,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder-15B' (link https://huggingface.co/bigcode/starcoder; direct code completion); pass@1 humaneval+ = 29.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder",
    "bench": "mbpp-plus",
    "score": 0.461,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder-15B' (link https://huggingface.co/bigcode/starcoder; direct code completion); pass@1 mbpp+ = 46.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b",
    "bench": "humaneval",
    "score": 0.463,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-15B' (link https://huggingface.co/bigcode/starcoder2-15b; direct code completion); pass@1 humaneval = 46.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b",
    "bench": "humaneval-plus",
    "score": 0.378,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-15B' (link https://huggingface.co/bigcode/starcoder2-15b; direct code completion); pass@1 humaneval+ = 37.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-3b",
    "bench": "humaneval",
    "score": 0.317,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-3B' (link https://huggingface.co/bigcode/starcoder2-3b; direct code completion); pass@1 humaneval = 31.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-3b",
    "bench": "humaneval-plus",
    "score": 0.274,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-3B' (link https://huggingface.co/bigcode/starcoder2-3b; direct code completion); pass@1 humaneval+ = 27.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-7b",
    "bench": "humaneval",
    "score": 0.354,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-7B' (link https://huggingface.co/bigcode/starcoder2-7b; direct code completion); pass@1 humaneval = 35.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-7b",
    "bench": "humaneval-plus",
    "score": 0.299,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoder2-7B' (link https://huggingface.co/bigcode/starcoder2-7b; direct code completion); pass@1 humaneval+ = 29.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humaneval",
    "score": 0.177,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-3B' (link https://huggingface.co/bigcode/starcoderbase-3b; direct code completion); pass@1 humaneval = 17.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humaneval-plus",
    "score": 0.159,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-3B' (link https://huggingface.co/bigcode/starcoderbase-3b; direct code completion); pass@1 humaneval+ = 15.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humaneval",
    "score": 0.244,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-7B' (link https://huggingface.co/bigcode/starcoderbase-7b; direct code completion); pass@1 humaneval = 24.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humaneval-plus",
    "score": 0.213,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-7B' (link https://huggingface.co/bigcode/starcoderbase-7b; direct code completion); pass@1 humaneval+ = 21.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-33b-v1-1",
    "bench": "humaneval",
    "score": 0.799,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-33B-V1.1' (link https://huggingface.co/WizardLM/WizardCoder-33B-V1.1; chat/instruction prompt); pass@1 humaneval = 79.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-33b-v1-1",
    "bench": "humaneval-plus",
    "score": 0.732,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-33B-V1.1' (link https://huggingface.co/WizardLM/WizardCoder-33B-V1.1; chat/instruction prompt); pass@1 humaneval+ = 73.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "humaneval",
    "score": 0.732,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-Python-34B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0; chat/instruction prompt); pass@1 humaneval = 73.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "humaneval-plus",
    "score": 0.646,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-Python-34B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0; chat/instruction prompt); pass@1 humaneval+ = 64.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "mbpp-plus",
    "score": 0.632,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-Python-34B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0; chat/instruction prompt); pass@1 mbpp+ = 63.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "humaneval",
    "score": 0.3,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Zephyr β-7B' (link https://huggingface.co/HuggingFaceH4/zephyr-7b-beta; direct code completion); pass@1 humaneval = 30.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "humaneval-plus",
    "score": 0.232,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Zephyr β-7B' (link https://huggingface.co/HuggingFaceH4/zephyr-7b-beta; direct code completion); pass@1 humaneval+ = 23.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "mbpp-plus",
    "score": 0.347,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Zephyr β-7B' (link https://huggingface.co/HuggingFaceH4/zephyr-7b-beta; direct code completion); pass@1 mbpp+ = 34.7 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-2b",
    "bench": "humaneval",
    "score": 0.268,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-2b' (link https://huggingface.co/google/codegemma-2b; direct code completion); pass@1 humaneval = 26.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-2b",
    "bench": "humaneval-plus",
    "score": 0.207,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-2b' (link https://huggingface.co/google/codegemma-2b; direct code completion); pass@1 humaneval+ = 20.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-2b",
    "bench": "mbpp-plus",
    "score": 0.466,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-2b' (link https://huggingface.co/google/codegemma-2b; direct code completion); pass@1 mbpp+ = 46.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b",
    "bench": "humaneval",
    "score": 0.445,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b' (link https://huggingface.co/google/codegemma-7b; direct code completion); pass@1 humaneval = 44.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b",
    "bench": "humaneval-plus",
    "score": 0.415,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b' (link https://huggingface.co/google/codegemma-7b; direct code completion); pass@1 humaneval+ = 41.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b",
    "bench": "mbpp-plus",
    "score": 0.524,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b' (link https://huggingface.co/google/codegemma-7b; direct code completion); pass@1 mbpp+ = 52.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humaneval",
    "score": 0.604,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b-it' (link https://huggingface.co/google/codegemma-7b-it; chat/instruction prompt); pass@1 humaneval = 60.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humaneval-plus",
    "score": 0.518,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b-it' (link https://huggingface.co/google/codegemma-7b-it; chat/instruction prompt); pass@1 humaneval+ = 51.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "mbpp-plus",
    "score": 0.569,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'codegemma-7b-it' (link https://huggingface.co/google/codegemma-7b-it; chat/instruction prompt); pass@1 mbpp+ = 56.9 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "humaneval",
    "score": 0.226,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-2b-it' (link https://huggingface.co/google/gemma-1.1-2b-it; chat/instruction prompt); pass@1 humaneval = 22.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "humaneval-plus",
    "score": 0.177,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-2b-it' (link https://huggingface.co/google/gemma-1.1-2b-it; chat/instruction prompt); pass@1 humaneval+ = 17.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "mbpp-plus",
    "score": 0.233,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-2b-it' (link https://huggingface.co/google/gemma-1.1-2b-it; chat/instruction prompt); pass@1 mbpp+ = 23.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "humaneval",
    "score": 0.427,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-7b-it' (link https://huggingface.co/google/gemma-1.1-7b-it; chat/instruction prompt); pass@1 humaneval = 42.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "humaneval-plus",
    "score": 0.354,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-7b-it' (link https://huggingface.co/google/gemma-1.1-7b-it; chat/instruction prompt); pass@1 humaneval+ = 35.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "mbpp-plus",
    "score": 0.45,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-1.1-7b-it' (link https://huggingface.co/google/gemma-1.1-7b-it; chat/instruction prompt); pass@1 mbpp+ = 45.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2b-base",
    "bench": "humaneval",
    "score": 0.25,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-2b' (link https://huggingface.co/google/gemma-2b; direct code completion); pass@1 humaneval = 25.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2b-base",
    "bench": "humaneval-plus",
    "score": 0.207,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-2b' (link https://huggingface.co/google/gemma-2b; direct code completion); pass@1 humaneval+ = 20.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2b-base",
    "bench": "mbpp-plus",
    "score": 0.341,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-2b' (link https://huggingface.co/google/gemma-2b; direct code completion); pass@1 mbpp+ = 34.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2b",
    "bench": "humaneval",
    "score": 0.177,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-2b-it' (link https://huggingface.co/google/gemma-2b-it; chat/instruction prompt); pass@1 humaneval = 17.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-2b",
    "bench": "humaneval-plus",
    "score": 0.152,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-2b-it' (link https://huggingface.co/google/gemma-2b-it; chat/instruction prompt); pass@1 humaneval+ = 15.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b-base",
    "bench": "humaneval",
    "score": 0.354,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b' (link https://huggingface.co/google/gemma-7b; direct code completion); pass@1 humaneval = 35.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b-base",
    "bench": "humaneval-plus",
    "score": 0.287,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b' (link https://huggingface.co/google/gemma-7b; direct code completion); pass@1 humaneval+ = 28.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b-base",
    "bench": "mbpp-plus",
    "score": 0.434,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b' (link https://huggingface.co/google/gemma-7b; direct code completion); pass@1 mbpp+ = 43.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b-it' (link https://huggingface.co/google/gemma-7b-it; chat/instruction prompt); pass@1 humaneval = 28.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b",
    "bench": "humaneval-plus",
    "score": 0.25,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b-it' (link https://huggingface.co/google/gemma-7b-it; chat/instruction prompt); pass@1 humaneval+ = 25.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemma-7b",
    "bench": "mbpp-plus",
    "score": 0.368,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'gemma-7b-it' (link https://huggingface.co/google/gemma-7b-it; chat/instruction prompt); pass@1 mbpp+ = 36.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phi-2",
    "bench": "humaneval",
    "score": 0.494,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'phi-2-2.7B' (link https://huggingface.co/microsoft/phi-2; direct code completion); pass@1 humaneval = 49.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phi-2",
    "bench": "humaneval-plus",
    "score": 0.451,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'phi-2-2.7B' (link https://huggingface.co/microsoft/phi-2; direct code completion); pass@1 humaneval+ = 45.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "phi-2",
    "bench": "mbpp-plus",
    "score": 0.542,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'phi-2-2.7B' (link https://huggingface.co/microsoft/phi-2; direct code completion); pass@1 mbpp+ = 54.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "stable-code-3b",
    "bench": "humaneval",
    "score": 0.293,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'stable-code-3B' (link https://huggingface.co/stabilityai/stable-code-3b; direct code completion); pass@1 humaneval = 29.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "stable-code-3b",
    "bench": "humaneval-plus",
    "score": 0.256,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'stable-code-3B' (link https://huggingface.co/stabilityai/stable-code-3b; direct code completion); pass@1 humaneval+ = 25.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "stable-code-3b",
    "bench": "mbpp-plus",
    "score": 0.458,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'stable-code-3B' (link https://huggingface.co/stabilityai/stable-code-3b; direct code completion); pass@1 mbpp+ = 45.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dbrx-instruct",
    "bench": "humaneval",
    "score": 0.75,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'databricks/dbrx-instruct' (link https://huggingface.co/databricks/dbrx-instruct; chat/instruction prompt); pass@1 humaneval = 75.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dbrx-instruct",
    "bench": "humaneval-plus",
    "score": 0.701,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'databricks/dbrx-instruct' (link https://huggingface.co/databricks/dbrx-instruct; chat/instruction prompt); pass@1 humaneval+ = 70.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dbrx-instruct",
    "bench": "mbpp-plus",
    "score": 0.558,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'databricks/dbrx-instruct' (link https://huggingface.co/databricks/dbrx-instruct; chat/instruction prompt); pass@1 mbpp+ = 55.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "humaneval",
    "score": 0.64,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Command-R+' (link https://huggingface.co/CohereForAI/c4ai-command-r-plus; chat/instruction prompt); pass@1 humaneval = 64 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "humaneval-plus",
    "score": 0.567,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Command-R+' (link https://huggingface.co/CohereForAI/c4ai-command-r-plus; chat/instruction prompt); pass@1 humaneval+ = 56.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "mbpp-plus",
    "score": 0.635,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Command-R+' (link https://huggingface.co/CohereForAI/c4ai-command-r-plus; chat/instruction prompt); pass@1 mbpp+ = 63.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humaneval",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-70B-Instruct' (link https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf; chat/instruction prompt); pass@1 humaneval = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humaneval-plus",
    "score": 0.659,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-70B-Instruct' (link https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf; chat/instruction prompt); pass@1 humaneval+ = 65.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mathstral-7b-v0-1",
    "bench": "mmlu-pro",
    "score": 0.41996343085106386,
    "source": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/eval_results/model_outputs_mathstral-7B_5shots.zip",
    "by": "independent",
    "note": "Published entry 'mathstral-7B' (Mathstral 7B v0.1, the only release). Author-released 5-shot CoT predictions (model_outputs_mathstral-7B_5shots.json, file dated 2024-07-20), 5053/12032 correct; compare stored pred to answer, including missing predictions as incorrect (12032 predictions stored, none missing). Inference dates and weight revision hashes not supplied. Published evaluator uses vLLM and temperature=0; actual invocation/token limit not recorded in archive. No category-level scores are also imported. Apache-2.0 (repository LICENSE), as for the existing mmlu-pro-author-runs files.",
    "source_id": "mmlu-pro-author-runs"
  },
  {
    "model": "devstral-2",
    "bench": "swe-bench-verified",
    "score": 0.722,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own Devstral 2 (123B, Instruct-2512) row in the Benchmark table of the pinned Apache-2.0 Devstral Small 2 card; same table, harness and footnote as the existing Devstral Small 2 rows from this card (the footnote marks only competitor rows as publicly reported values; Devstral 2 is Mistral's own model). Competitor rows not imported. Scaffold and decoding not tied to the table; none assumed. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512"
  },
  {
    "model": "devstral-2",
    "bench": "swe-bench-multilingual",
    "score": 0.613,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own Devstral 2 (123B, Instruct-2512) row in the Benchmark table of the pinned Apache-2.0 Devstral Small 2 card; same table, harness and footnote as the existing Devstral Small 2 rows from this card (the footnote marks only competitor rows as publicly reported values; Devstral 2 is Mistral's own model). Competitor rows not imported. Scaffold and decoding not tied to the table; none assumed. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512"
  },
  {
    "model": "devstral-2",
    "bench": "terminal-bench-2",
    "score": 0.326,
    "source": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512/blob/55c5b41e98c2dbd21b0c8afffc540dcfc9eb5128/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own Devstral 2 (123B, Instruct-2512) row in the Benchmark table of the pinned Apache-2.0 Devstral Small 2 card; same table, harness and footnote as the existing Devstral Small 2 rows from this card (the footnote marks only competitor rows as publicly reported values; Devstral 2 is Mistral's own model). Competitor rows not imported. Scaffold and decoding not tied to the table; none assumed. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-devstral-small-2-24b-instruct-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2025",
    "score": 0.85,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 14B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2024",
    "score": 0.898,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 14B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.712,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 14B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.646,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 14B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "aime-2025",
    "score": 0.721,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 3B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "aime-2024",
    "score": 0.775,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 3B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.534,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 3B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.548,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512/blob/81eaece1948f3875421d9a45bc55487d10e2d894/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral's own 'Ministral 3 3B' row in the Benchmark Results / Reasoning table of the pinned Ministral 3 8B Reasoning 2512 card (Reasoning section = the Reasoning 2512 checkpoints, BF16); same table as the existing Ministral 3 8B Reasoning rows. Values already fractions. Qwen comparison rows not imported. LiveCodeBench version/window and per-task decoding not specified. Review date is not the evaluation-run date.",
    "source_id": "card-mistralai-ministral-3-8b-reasoning-2512"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-diamond",
    "score": 0.3693,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'GPQA Diamond' = 36.93 (± 2.07); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-pro",
    "score": 0.5668,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'MMLU-Pro' = 56.68 (± 1.29); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifeval-mean-four",
    "score": 0.7165,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'IFEval' = 71.65 (± 0.13); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifbench-unspecified",
    "score": 0.2588,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'IFBench' = 25.88 (± 0.30); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gsm8k",
    "score": 0.856,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'GSM8K' = 85.60 (± 1.13); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "math-500",
    "score": 0.8192,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'MATH-500' = 81.92 (± 2.99); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "aime-2025",
    "score": 0.3627,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'AIME25' = 36.27 (± 1.24); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bfcl-v3",
    "score": 0.5541,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Qwen3-1.7B (thinking mode)', column 'BFCLv3' = 55.41 (± 0.04); the card states its methodology for every row (mean of five runs at temperature 0.6 (± is the standard deviation); 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gpqa-diamond",
    "score": 0.2424,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'GPQA Diamond' = 24.24; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mmlu-pro",
    "score": 0.3353,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'MMLU-Pro' = 33.53; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "ifeval-mean-four",
    "score": 0.7961,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'IFEval' = 79.61; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "ifbench-unspecified",
    "score": 0.21,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'IFBench' = 21; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gsm8k",
    "score": 0.7342,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'GSM8K' = 73.42; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "math-500",
    "score": 0.448,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'MATH-500' = 44.80; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "aime-2025",
    "score": 0.0333,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'AIME25' = 3.33; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "bfcl-v3",
    "score": 0.5243,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-1B', column 'BFCLv3' = 52.43; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gpqa-diamond",
    "score": 0.2424,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'GPQA Diamond' = 24.24; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmlu-pro",
    "score": 0.1404,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'MMLU-Pro' = 14.04; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifeval-mean-four",
    "score": 0.6325,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'IFEval' = 63.25; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifbench-unspecified",
    "score": 0.2047,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'IFBench' = 20.47; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gsm8k",
    "score": 0.4215,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'GSM8K' = 42.15; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "math-500",
    "score": 0.452,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'MATH-500' = 45.20; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "aime-2025",
    "score": 0.01,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'AIME25' = 1; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "gemma-3-1b",
    "bench": "bfcl-v3",
    "score": 0.1664,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Gemma 3 1B IT', column 'BFCLv3' = 16.64; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gpqa-diamond",
    "score": 0.1657,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'GPQA Diamond' = 16.57; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking",
    "invalid": "Below guessing in Liquid AI's greedy-decoding comparison run; Epoch measures the same checkpoint at 0.239."
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmlu-pro",
    "score": 0.208,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'MMLU-Pro' = 20.80; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifeval-mean-four",
    "score": 0.5237,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'IFEval' = 52.37; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifbench-unspecified",
    "score": 0.1593,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'IFBench' = 15.93; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gsm8k",
    "score": 0.3904,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'GSM8K' = 39.04; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "math-500",
    "score": 0.234,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'MATH-500' = 23.40; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "aime-2025",
    "score": 0.0033,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'AIME25' = 0.33; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bfcl-v3",
    "score": 0.2144,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Llama 3.2 1B Instruct', column 'BFCLv3' = 21.44; the card states its methodology for every row (greedy decoding; 'For instruct models, we report scores using greedy decoding'). Same table as the existing LFM2.5-1.2B-Thinking rows. IFEval is the mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 follow the cited AA methodology but are Liquid's own runs. Multi-IF column not used. Review date is not the evaluation-run date.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-pro",
    "score": 0.848,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'MMLU-Pro', column 'DeepSeek-V3.1' = 84.8. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "mmlu-pro",
    "score": 0.85,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'MMLU-Pro', column 'DeepSeek-V3.1-Terminus' = 85.0. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "gpqa-diamond",
    "score": 0.801,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'GPQA-Diamond', column 'DeepSeek-V3.1' = 80.1. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "gpqa-diamond",
    "score": 0.807,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'GPQA-Diamond', column 'DeepSeek-V3.1-Terminus' = 80.7. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "hle",
    "score": 0.159,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'Humanity's Last Exam', column 'DeepSeek-V3.1' = 15.9. HLE without tools. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "hle",
    "score": 0.217,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'Humanity's Last Exam', column 'DeepSeek-V3.1-Terminus' = 21.7. HLE without tools. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "livecodebench-unversioned",
    "score": 0.748,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'LiveCodeBench', column 'DeepSeek-V3.1' = 74.8. Card does not state the LiveCodeBench window, so the unversioned id is used. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "livecodebench-unversioned",
    "score": 0.749,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'LiveCodeBench', column 'DeepSeek-V3.1-Terminus' = 74.9. Card does not state the LiveCodeBench window, so the unversioned id is used. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aider-polyglot",
    "score": 0.763,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'Aider-Polyglot', column 'DeepSeek-V3.1' = 76.3. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "aider-polyglot",
    "score": 0.761,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Reasoning Mode w/o Tool Use', row 'Aider-Polyglot', column 'DeepSeek-V3.1-Terminus' = 76.1. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "browsecomp",
    "score": 0.3,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'BrowseComp', column 'DeepSeek-V3.1' = 30.0. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "browsecomp",
    "score": 0.385,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'BrowseComp', column 'DeepSeek-V3.1-Terminus' = 38.5. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "browsecomp-zh",
    "score": 0.492,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'BrowseComp-zh', column 'DeepSeek-V3.1' = 49.2. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "browsecomp-zh",
    "score": 0.45,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'BrowseComp-zh', column 'DeepSeek-V3.1-Terminus' = 45.0. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "swe-bench-verified",
    "score": 0.66,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'SWE Verified', column 'DeepSeek-V3.1' = 66.0. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "swe-bench-verified",
    "score": 0.684,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'SWE Verified', column 'DeepSeek-V3.1-Terminus' = 68.4. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "swe-bench-multilingual",
    "score": 0.545,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'SWE-bench Multilingual', column 'DeepSeek-V3.1' = 54.5. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "swe-bench-multilingual",
    "score": 0.578,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'SWE-bench Multilingual', column 'DeepSeek-V3.1-Terminus' = 57.8. SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "terminal-bench-pre-2",
    "score": 0.313,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'Terminal-bench', column 'DeepSeek-V3.1' = 31.3. Terminal-Bench task-set version not stated (card dated September 2025, before Terminal-Bench 2.0). SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "deepseek-v3-1-terminus",
    "bench": "terminal-bench-pre-2",
    "score": 0.367,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus/blob/19510d6dc61f79dbd925bd51ee8a9081c509a4b6/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "DeepSeek's own evaluation in the pinned MIT DeepSeek-V3.1-Terminus model card, comparison table, section 'Agentic Tool Use', row 'Terminal-bench', column 'DeepSeek-V3.1-Terminus' = 36.7. Terminal-Bench task-set version not stated (card dated September 2025, before Terminal-Bench 2.0). SimpleQA (search-agent setting) and Codeforces rows not used. The card notes the search-agent template and tool set were updated for Terminus. Date is the release date (repository commit 'Release DeepSeek-V3.1-Terminus', 2025-09-22).",
    "source_id": "card-deepseek-ai-deepseek-v3-1-terminus"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "aime-2024",
    "score": 0.768,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "aime-2025",
    "score": 0.725,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "bbh",
    "score": 0.898,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "zebralogic",
    "score": 0.76,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "agieval-en",
    "score": 0.882,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "humaneval-plus",
    "score": 0.914,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "mbpp-plus",
    "score": 0.68,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "livecodebench-v3",
    "score": 0.835,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "ifeval-unspecified",
    "score": 0.89,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "ifbench-unspecified",
    "score": 0.476,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "mmlu",
    "score": 0.854,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "popqa",
    "score": 0.319,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "gpqa-unspecified",
    "score": 0.581,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "alpacaeval2-lc",
    "score": 0.742,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Table 14 'Final Think 3.0' column (the original Olmo-3-32B-Think release; 'Final Think 3.1' is the separate Olmo 3.1 model), not SFT/DPO. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "aime-2024",
    "score": 0.677,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "aime-2025",
    "score": 0.572,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "bbh",
    "score": 0.771,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "zebralogic",
    "score": 0.349,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "agieval-en",
    "score": 0.786,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "humaneval-plus",
    "score": 0.874,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "mbpp-plus",
    "score": 0.614,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "livecodebench-v3",
    "score": 0.68,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "ifeval-unspecified",
    "score": 0.517,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "ifbench-unspecified",
    "score": 0.23,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "mmlu",
    "score": 0.774,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "popqa",
    "score": 0.18,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "gpqa-unspecified",
    "score": 0.476,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "openthinker3-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.24,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'OpenThinker3 7B', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2024",
    "score": 0.721,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2025",
    "score": 0.589,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "bbh",
    "score": 0.862,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "zebralogic",
    "score": 0.608,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "agieval-en",
    "score": 0.831,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "humaneval-plus",
    "score": 0.897,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "mbpp-plus",
    "score": 0.661,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-v3",
    "score": 0.834,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "ifeval-unspecified",
    "score": 0.86,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "ifbench-unspecified",
    "score": 0.346,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "mmlu",
    "score": 0.843,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "popqa",
    "score": 0.179,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "gpqa-unspecified",
    "score": 0.562,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "alpacaeval2-lc",
    "score": 0.58,
    "source": "https://arxiv.org/html/2512.13961v2",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 15 baseline column 'Nemotron Nano 9B v2', run by Ai2 with its own evaluation framework (caption: 'We evaluate all models using our evaluation framework'; section 4.1.1: 'a standard evaluation configuration between all baseline models'); all numbers mean of three runs; reasoning setting per the shared thinking-model configuration. Table 16: zero-shot CoT, temperature 0.6, top_p 0.95, max tokens 32768; AIME 32 samples, code 10; thinking traces stripped before scoring. GPQA subset and IFEval/IFBench aggregation are not explicitly stated. MATH, OMEGA and Safety rows not used (as for the existing rows from this report). Review date is not the evaluation-run date.",
    "source_id": "report-2512.13961v2"
  },
  {
    "model": "devstral-2",
    "bench": "dseffi-bench-pass",
    "score": 0.43,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Devstral-2-123B', Overall Pass = 43.0% (greedy T=0.0). Identifier per Appendix C.2: mistralai/Devstral-2-123B-Instruct-2512 served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "dseffi-bench-pass",
    "score": 0.328,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Qwen2.5-Coder-14B', Overall Pass = 32.8% (greedy T=0.0). Identifier per Appendix C.2: Qwen/Qwen2.5-Coder-14B-Instruct served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "codestral-22b-v0-1",
    "bench": "dseffi-bench-pass",
    "score": 0.274,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'Codestral-22B', Overall Pass = 27.4% (greedy T=0.0). Identifier per Appendix C.2: mistralai/Codestral-22B-v0.1 served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "dseffi-bench-pass",
    "score": 0.175,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'DeepSeek-Coder-33B', Overall Pass = 17.5% (greedy T=0.0). Identifier per Appendix C.2: deepseek-ai/deepseek-coder-33b-instruct served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "dseffi-bench-pass",
    "score": 0.171,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'DeepSeek-Coder-V2-Lite', Overall Pass = 17.1% (greedy T=0.0). Identifier per Appendix C.2: deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "dseffi-bench-pass",
    "score": 0.577,
    "source": "https://arxiv.org/html/2608.30248v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "DSEffi-Bench (arXiv:2608.30248v1, CC BY 4.0) Table 2, row 'DeepSeek-V4-Flash', Overall Pass = 57.7% (greedy T=0.0). Identifier per Appendix C.2: deepseek-ai/DeepSeek-V4-Flash served via vLLM. Reasoning/thinking settings not stated (provider/serving defaults). Authors' own run. Same table as the existing rows from this paper.",
    "source_id": "paper-2608.30248v1"
  },
  {
    "model": "gpt-5-pro",
    "bench": "gpqa-diamond",
    "score": 0.884,
    "source": "https://openai.com/index/introducing-gpt-5",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT-5' (August 7, 2025); read from Internet Archive capture 20261004073440 of the page (openai.com returns HTTP 403 to direct fetches). Chart 'GPQA Diamond' bar 'GPT-5 pro (no tools)' labelled 88.4; the prose also states 'GPT-5 pro ... scoring 88.4% without tools'. GPT-5 pro uses scaled parallel test-time compute; the API model gpt-5-pro (2025-10-06) is OpenAI's API release of GPT-5 pro. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-pro",
    "bench": "hle",
    "score": 0.307,
    "source": "https://openai.com/index/introducing-gpt-5",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT-5' (August 7, 2025); read from Internet Archive capture 20261004073440 of the page (openai.com returns HTTP 403 to direct fetches). Chart 'Humanity's Last Exam (Full Set)*' bar 'GPT-5 pro (no tools)' labelled 30.7 (footnote: small discrepancy vs earlier posts run on a former HLE version). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-pro",
    "bench": "hle-with-tools",
    "score": 0.42,
    "source": "https://openai.com/index/introducing-gpt-5",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT-5' (August 7, 2025); read from Internet Archive capture 20261004073440 of the page (openai.com returns HTTP 403 to direct fetches). Chart 'Humanity's Last Exam (Full Set)*' bar 'GPT-5 pro (python + search with blocklist)' labelled 42. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5",
    "bench": "gpqa-diamond",
    "score": 0.857,
    "source": "https://openai.com/index/introducing-gpt-5",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT-5' (August 7, 2025); read from Internet Archive capture 20261004073440 of the page (openai.com returns HTTP 403 to direct fetches). Chart 'GPQA Diamond' bar 'GPT-5 (no tools)', group 'With thinking', labelled 85.7 (the 'Without thinking' bar, 77.8, not used). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o3-pro",
    "bench": "gpqa-diamond",
    "score": 0.84,
    "source": "https://help.openai.com/en/articles/9624314-model-release-notes",
    "by": "lab",
    "date": "2025-06-10",
    "note": "OpenAI Help Center 'Model Release Notes', entry 'Launching OpenAI o3-pro' (June 10, 2025), chart image 'OpenAI o3-pro: pass@1 evaluation benchmarks'; read from Internet Archive capture 20250616141951 of the page (chart image from the same capture). Chart methodology note 1: 'Evals were run for all models using default (medium) ChatGPT thinking time.' Panel 'PhD Science Questions (GPQA Diamond)', bar 'o3-pro' labelled 84% (pass@1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o3-pro",
    "bench": "aime-2024",
    "score": 0.93,
    "source": "https://help.openai.com/en/articles/9624314-model-release-notes",
    "by": "lab",
    "date": "2025-06-10",
    "note": "OpenAI Help Center 'Model Release Notes', entry 'Launching OpenAI o3-pro' (June 10, 2025), chart image 'OpenAI o3-pro: pass@1 evaluation benchmarks'; read from Internet Archive capture 20250616141951 of the page (chart image from the same capture). Chart methodology note 1: 'Evals were run for all models using default (medium) ChatGPT thinking time.' Panel 'Competition Math (AIME 2024)', bar 'o3-pro' labelled 93% (pass@1). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o3",
    "bench": "gpqa-diamond",
    "score": 0.81,
    "source": "https://help.openai.com/en/articles/9624314-model-release-notes",
    "by": "lab",
    "date": "2025-06-10",
    "note": "OpenAI Help Center 'Model Release Notes', entry 'Launching OpenAI o3-pro' (June 10, 2025), chart image 'OpenAI o3-pro: pass@1 evaluation benchmarks'; read from Internet Archive capture 20250616141951 of the page (chart image from the same capture). Chart methodology note 1: 'Evals were run for all models using default (medium) ChatGPT thinking time.' Panel 'PhD Science Questions (GPQA Diamond)', bar 'o3(medium)' labelled 81% (pass@1); medium ChatGPT thinking time, not o3's highest setting. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o3",
    "bench": "aime-2024",
    "score": 0.9,
    "source": "https://help.openai.com/en/articles/9624314-model-release-notes",
    "by": "lab",
    "date": "2025-06-10",
    "note": "OpenAI Help Center 'Model Release Notes', entry 'Launching OpenAI o3-pro' (June 10, 2025), chart image 'OpenAI o3-pro: pass@1 evaluation benchmarks'; read from Internet Archive capture 20250616141951 of the page (chart image from the same capture). Chart methodology note 1: 'Evals were run for all models using default (medium) ChatGPT thinking time.' Panel 'Competition Math (AIME 2024)', bar 'o3(medium)' labelled 90% (pass@1); medium ChatGPT thinking time, not o3's highest setting. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1-5",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.619,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Llama-3.3-Nemotron-Super-49B-v1.5' (NVIDIA's own released model), column 'LiveCodeBench Pro 25Q2 Easy' = 61.9: LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode. Appendix B: 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.' Same table and columns as the existing NVIDIA-Nemotron-Nano-9B-v2 rows.",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1-5",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.061,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Llama-3.3-Nemotron-Super-49B-v1.5' (NVIDIA's own released model), column 'LiveCodeBench Pro 25Q2 Med' = 6.1: LiveCodeBench Pro 2025Q2 split (2025/04-2025/07, 167 problems), pass@1 avg@8 in thinking mode. Appendix B: 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.' Same table and columns as the existing NVIDIA-Nemotron-Nano-9B-v2 rows.",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "mmlu",
    "score": 0.831,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'MMLU(0 shot, CoT){}_{\\text{(0 shot, CoT)}}' = 83.1 (divided by 100). MMLU, 0-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "popqa",
    "score": 0.465,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'PopQA(15 shot){}_{\\text{(15 shot)}}' = 46.5 (divided by 100). PopQA, 15-shot, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "truthfulqa-mc2",
    "score": 0.676,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'TruthfulQA(6 shot){}_{\\text{(6 shot)}}' = 67.6 (divided by 100). TruthfulQA, 6-shot, MC2 (Table 24 metric); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "bbh",
    "score": 0.85,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'BigBenchHard(3 shot, CoT){}_{\\text{(3 shot, CoT)}}' = 85.0 (divided by 100). BigBenchHard, 3-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "drop",
    "score": 0.743,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'DROP(3 shot){}_{\\text{(3 shot)}}' = 74.3 (divided by 100). DROP, 3-shot, F1; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "math-subset-unspecified",
    "score": 0.63,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'MATH(4 shot CoT, Flex){}_{\\text{(4 shot CoT, Flex)}}' = 63.0 (divided by 100). MATH, 4-shot CoT, Flex EM (subset not resolved, as in the OLMo 2 paper rows from the same OLMES suite); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "gsm8k",
    "score": 0.935,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'GSM8K(8 shot, CoT){}_{\\text{(8 shot, CoT)}}' = 93.5 (divided by 100). GSM8K, 8-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "ifeval-prompt-loose",
    "score": 0.832,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'IFEval(prompt loose){}_{\\text{(prompt loose)}}' = 83.2 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.498,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B', row 'AlpacaEval 2(LC % win){}_{\\text{(LC \\% win)}}' = 49.8 (divided by 100). AlpacaEval 2, LC win rate; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "mmlu",
    "score": 0.833,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'MMLU(0 shot, CoT){}_{\\text{(0 shot, CoT)}}' = 83.3 (divided by 100). MMLU, 0-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "popqa",
    "score": 0.463,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'PopQA(15 shot){}_{\\text{(15 shot)}}' = 46.3 (divided by 100). PopQA, 15-shot, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "truthfulqa-mc2",
    "score": 0.679,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'TruthfulQA(6 shot){}_{\\text{(6 shot)}}' = 67.9 (divided by 100). TruthfulQA, 6-shot, MC2 (Table 24 metric); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "bbh",
    "score": 0.848,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'BigBenchHard(3 shot, CoT){}_{\\text{(3 shot, CoT)}}' = 84.8 (divided by 100). BigBenchHard, 3-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "drop",
    "score": 0.741,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'DROP(3 shot){}_{\\text{(3 shot)}}' = 74.1 (divided by 100). DROP, 3-shot, F1; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "math-subset-unspecified",
    "score": 0.623,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'MATH(4 shot CoT, Flex){}_{\\text{(4 shot CoT, Flex)}}' = 62.3 (divided by 100). MATH, 4-shot CoT, Flex EM (subset not resolved, as in the OLMo 2 paper rows from the same OLMES suite); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "gsm8k",
    "score": 0.935,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'GSM8K(8 shot, CoT){}_{\\text{(8 shot, CoT)}}' = 93.5 (divided by 100). GSM8K, 8-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "ifeval-prompt-loose",
    "score": 0.826,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'IFEval(prompt loose){}_{\\text{(prompt loose)}}' = 82.6 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-tulu-3-70b-dpo",
    "bench": "alpacaeval2-lc",
    "score": 0.496,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Tülu 3 70B DPO', row 'AlpacaEval 2(LC % win){}_{\\text{(LC \\% win)}}' = 49.6 (divided by 100). AlpacaEval 2, LC win rate; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "mmlu",
    "score": 0.838,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'MMLU(0 shot, CoT){}_{\\text{(0 shot, CoT)}}' = 83.8 (divided by 100). MMLU, 0-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "popqa",
    "score": 0.364,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'PopQA(15 shot){}_{\\text{(15 shot)}}' = 36.4 (divided by 100). PopQA, 15-shot, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "truthfulqa-mc2",
    "score": 0.626,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'TruthfulQA(6 shot){}_{\\text{(6 shot)}}' = 62.6 (divided by 100). TruthfulQA, 6-shot, MC2 (Table 24 metric); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "bbh",
    "score": 0.785,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'BigBenchHard(3 shot, CoT){}_{\\text{(3 shot, CoT)}}' = 78.5 (divided by 100). BigBenchHard, 3-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "drop",
    "score": 0.688,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'DROP(3 shot){}_{\\text{(3 shot)}}' = 68.8 (divided by 100). DROP, 3-shot, F1; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "math-subset-unspecified",
    "score": 0.55,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'MATH(4 shot CoT, Flex){}_{\\text{(4 shot CoT, Flex)}}' = 55.0 (divided by 100). MATH, 4-shot CoT, Flex EM (subset not resolved, as in the OLMo 2 paper rows from the same OLMES suite); Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "gsm8k",
    "score": 0.847,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'GSM8K(8 shot, CoT){}_{\\text{(8 shot, CoT)}}' = 84.7 (divided by 100). GSM8K, 8-shot CoT, EM; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "ifeval-prompt-loose",
    "score": 0.799,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'IFEval(prompt loose){}_{\\text{(prompt loose)}}' = 79.9 (divided by 100). IFEval, 0-shot, pass@1 prompt-level loose; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.661,
    "source": "https://arxiv.org/html/2411.15124v5",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Tülu 3 paper (Lambert et al., arXiv:2411.15124v5, CC BY 4.0), Table 5 'Summary of Tülu 3 results relative to peer 70B models', column 'Nemotron Llama 3.1 70B', row 'AlpacaEval 2(LC % win){}_{\\text{(LC \\% win)}}' = 66.1 (divided by 100). AlpacaEval 2, LC win rate; Tülu 3 Eval (OLMES) settings per Table 24, run by Ai2 for every column. Llama 3.1 70B Instruct and Qwen 2.5 72B Instruct columns not added: the same Ai2 numbers are already recorded from the OLMo 2 paper. HumanEval/HumanEval+ (pass@10) and Safety rows not used.",
    "source_id": "paper-2411.15124v5"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 59.2982,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gpt-4o-2024-05-13': WB_score.task_macro = 59.2982 (WB_score 58.8074; raw mean checklist score 7.9404 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 55.2212,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gpt-4-turbo-2024-04-09': WB_score.task_macro = 55.2212 (WB_score 56.0899; raw mean checklist score 7.8045 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 52.2775,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gpt-4-0125-preview': WB_score.task_macro = 52.2775 (WB_score 53.2812; raw mean checklist score 7.6641 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 51.714,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'claude-3-opus-20240229': WB_score.task_macro = 51.7140 (WB_score 52.1094; raw mean checklist score 7.6055 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-3-70b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 47.7708,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Meta-Llama-3-70B-Instruct': WB_score.task_macro = 47.7708 (WB_score 49.5797; raw mean checklist score 7.4790 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 39.9277,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Qwen1.5-72B-Chat-greedy': WB_score.task_macro = 39.9277 (WB_score 43.4672; raw mean checklist score 7.1734 over 1021 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Entry run with greedy decoding. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 45.4815,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'claude-3-sonnet-20240229': WB_score.task_macro = 45.4815 (WB_score 45.2446; raw mean checklist score 7.2622 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "mistral-large-2402",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 38.8937,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'mistral-large-2402': WB_score.task_macro = 38.8937 (WB_score 42.2874; raw mean checklist score 7.1144 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 38.8936,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'claude-3-haiku-20240307': WB_score.task_macro = 38.8936 (WB_score 40.2539; raw mean checklist score 7.0127 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "dbrx-instruct",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 32.5989,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'dbrx-instruct@together': WB_score.task_macro = 32.5989 (WB_score 35.5425; raw mean checklist score 6.7771 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Served by Together AI. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 31.4703,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Mixtral-8x7B-Instruct-v0.1': WB_score.task_macro = 31.4703 (WB_score 35.0294; raw mean checklist score 6.7515 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 30.1694,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Starling-LM-7B-beta': WB_score.task_macro = 30.1694 (WB_score 34.1740; raw mean checklist score 6.7087 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "command-r-03-2024",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 29.5331,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'command-r': WB_score.task_macro = 29.5331 (WB_score 35.0586; raw mean checklist score 6.7529 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Data committed by 2024-08-01, before the 08-2024 Command R release, so the API name command-r is the 03-2024 model. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 36.7624,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'command-r-plus': WB_score.task_macro = 36.7624 (WB_score 41.5656; raw mean checklist score 7.0783 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Data committed by 2024-08-01, before the 08-2024 release, so command-r-plus is the 04-2024 model. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-3-8b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 29.2028,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Meta-Llama-3-8B-Instruct': WB_score.task_macro = 29.2028 (WB_score 33.1769; raw mean checklist score 6.6588 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 27.9838,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "lab",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'tulu-2-dpo-70b': WB_score.task_macro = 27.9838 (WB_score 32.8250; raw mean checklist score 6.6413 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "nous-hermes-2-mixtral-8x7b-dpo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 30.7114,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Nous-Hermes-2-Mixtral-8x7B-DPO': WB_score.task_macro = 30.7114 (WB_score 33.2223; raw mean checklist score 6.6611 over 1021 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 25.6337,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Mistral-7B-Instruct-v0.2': WB_score.task_macro = 25.6337 (WB_score 30.6940; raw mean checklist score 6.5347 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 30.016,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gpt-3.5-turbo-0125': WB_score.task_macro = 30.0160 (WB_score 32.2776; raw mean checklist score 6.6139 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 23.4232,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Qwen1.5-7B-Chat@together': WB_score.task_macro = 23.4232 (WB_score 27.3705; raw mean checklist score 6.3685 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Served by Together AI. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 20.6596,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-2-70b-chat-hf': WB_score.task_macro = 20.6596 (WB_score 26.9141; raw mean checklist score 6.3457 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 8.2621,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-2-7b-chat-hf': WB_score.task_macro = 8.2621 (WB_score 15.2250; raw mean checklist score 5.7613 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-7b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 6.6198,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-7b-it': WB_score.task_macro = 6.6198 (WB_score 10.1758; raw mean checklist score 5.5088 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": -9.6919,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2b-it': WB_score.task_macro = -9.6919 (WB_score -5.2498; raw mean checklist score 4.7375 over 1021 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "qwen2-72b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 44.4977,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Qwen2-72B-Instruct': WB_score.task_macro = 44.4977 (WB_score 46.4062; raw mean checklist score 7.3203 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 45.6135,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Yi-1.5-34B-Chat': WB_score.task_macro = 45.6135 (WB_score 47.3509; raw mean checklist score 7.3675 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "nemotron-4-340b-instruct",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 47.6725,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'nemotron-4-340b-instruct': WB_score.task_macro = 47.6725 (WB_score 48.8477; raw mean checklist score 7.4424 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 54.6951,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'claude-3-5-sonnet-20240620': WB_score.task_macro = 54.6951 (WB_score 54.5312; raw mean checklist score 7.7266 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2-9b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 42.6962,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2-9b-it': WB_score.task_macro = 42.6962 (WB_score 45.3620; raw mean checklist score 7.2681 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "athene-70b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 59.5374,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Athene-70B': WB_score.task_macro = 59.5374 (WB_score 59.4129; raw mean checklist score 7.9706 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 57.1369,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gpt-4o-mini-2024-07-18': WB_score.task_macro = 57.1369 (WB_score 57.2656; raw mean checklist score 7.8633 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "mistral-large-2407",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 55.5683,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Mistral-Large-2': WB_score.task_macro = 55.5683 (WB_score 55.8008; raw mean checklist score 7.7900 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). 'Mistral-Large-2' in data committed by 2024-08-01 is the July 2024 model (Mistral-Large-Instruct-2407), not 2411. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2-9b-simpo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 53.2792,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2-9b-it-SimPO': WB_score.task_macro = 53.2792 (WB_score 54.0762; raw mean checklist score 7.7038 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2-27b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 48.5402,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2-27b-it@together': WB_score.task_macro = 48.5402 (WB_score 49.3945; raw mean checklist score 7.4697 over 1024 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Served by Together AI. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 44.3751,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Mistral-Nemo-Instruct-2407': WB_score.task_macro = 44.3751 (WB_score 46.8622; raw mean checklist score 7.3431 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 39.099,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'glm-4-9b-chat': WB_score.task_macro = 39.0990 (WB_score 41.1730; raw mean checklist score 7.0587 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 38.6654,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Yi-1.5-9B-Chat': WB_score.task_macro = 38.6654 (WB_score 39.8436; raw mean checklist score 6.9922 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2-2b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 27.826,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2-2b-it': WB_score.task_macro = 27.8260 (WB_score 32.7202; raw mean checklist score 6.6360 over 1022 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 23.3181,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Yi-1.5-6B-Chat': WB_score.task_macro = 23.3181 (WB_score 25.2786; raw mean checklist score 6.2639 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "zephyr-orpo-141b-a35b",
    "bench": "ifeval-unspecified",
    "score": 0.6506,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "lab",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'zephyr-orpo-141b-A39b-v0.1', column 'IFEval' = 65.06. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "zephyr-orpo-141b-a35b",
    "bench": "bbh",
    "score": 0.5896,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "lab",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'zephyr-orpo-141b-A39b-v0.1', column 'BBH' = 58.96. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "dbrx-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.5213,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "independent",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'databricks/dbrx-instruct', column 'IFEval' = 52.13. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "dbrx-instruct",
    "bench": "bbh",
    "score": 0.485,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "independent",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'databricks/dbrx-instruct', column 'BBH' = 48.50. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "ifeval-unspecified",
    "score": 0.5508,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "independent",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'mistralai/Mixtral-8x7B-Instruct-v0.1', column 'IFEval' = 55.08. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "bbh",
    "score": 0.4531,
    "source": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1/blob/a3be084543d278e61b64cd600f28157afc79ffd3/README.md",
    "by": "independent",
    "date": "2024-04-10",
    "note": "Hugging Face H4 Zephyr 141B-A39B (ORPO) model card, Performance table, row 'mistralai/Mixtral-8x7B-Instruct-v0.1', column 'BBH' = 45.31. Card: 'The scores reported below were obtained using the LightEval evaluation suite and each prompt has been formatted with the model's corresponding chat template', i.e. H4 ran all three rows itself. IFEval aggregation and BBH shot count not stated. MT Bench (no catalogue id) and AGIEval (subset not stated) columns not used. Apache-2.0 card. Date is the model repository creation date (2024-04-10).",
    "source_id": "card-huggingfaceh4-zephyr-orpo-141b-a35b-v0-1"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "mmlu",
    "score": 0.728,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MMLU (5-shot)', column 'InternLM2.5-7B-Chat' = 72.8. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "mmlu",
    "score": 0.684,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MMLU (5-shot)', column 'Llama3-8B-Instruct' = 68.4. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mmlu",
    "score": 0.709,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MMLU (5-shot)', column 'Gemma2-9B-IT' = 70.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "mmlu",
    "score": 0.708,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MMLU (5-shot)', column 'Qwen2-7B-Instruct' = 70.8. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "cmmlu",
    "score": 0.78,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'CMMLU (5-shot)', column 'InternLM2.5-7B-Chat' = 78.0. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "cmmlu",
    "score": 0.533,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'CMMLU (5-shot)', column 'Llama3-8B-Instruct' = 53.3. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "gemma-2-9b",
    "bench": "cmmlu",
    "score": 0.603,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'CMMLU (5-shot)', column 'Gemma2-9B-IT' = 60.3. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "cmmlu",
    "score": 0.809,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'CMMLU (5-shot)', column 'Qwen2-7B-Instruct' = 80.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "bbh",
    "score": 0.716,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'BBH (3-shot CoT)', column 'InternLM2.5-7B-Chat' = 71.6. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "bbh",
    "score": 0.544,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'BBH (3-shot CoT)', column 'Llama3-8B-Instruct' = 54.4. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "bbh",
    "score": 0.65,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'BBH (3-shot CoT)', column 'Qwen2-7B-Instruct' = 65.0. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "math-subset-unspecified",
    "score": 0.601,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MATH (0-shot CoT)', column 'InternLM2.5-7B-Chat' = 60.1. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.279,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MATH (0-shot CoT)', column 'Llama3-8B-Instruct' = 27.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "gemma-2-9b",
    "bench": "math-subset-unspecified",
    "score": 0.469,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MATH (0-shot CoT)', column 'Gemma2-9B-IT' = 46.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "math-subset-unspecified",
    "score": 0.486,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'MATH (0-shot CoT)', column 'Qwen2-7B-Instruct' = 48.6. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "gsm8k",
    "score": 0.86,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GSM8K (0-shot CoT)', column 'InternLM2.5-7B-Chat' = 86.0. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "gsm8k",
    "score": 0.729,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GSM8K (0-shot CoT)', column 'Llama3-8B-Instruct' = 72.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gsm8k",
    "score": 0.889,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GSM8K (0-shot CoT)', column 'Gemma2-9B-IT' = 88.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "gsm8k",
    "score": 0.829,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GSM8K (0-shot CoT)', column 'Qwen2-7B-Instruct' = 82.9. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "internlm2-5-7b-chat",
    "bench": "gpqa-unspecified",
    "score": 0.384,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "lab",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GPQA (0-shot)', column 'InternLM2.5-7B-Chat' = 38.4. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "llama-3-8b",
    "bench": "gpqa-unspecified",
    "score": 0.261,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GPQA (0-shot)', column 'Llama3-8B-Instruct' = 26.1. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gpqa-unspecified",
    "score": 0.338,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GPQA (0-shot)', column 'Gemma2-9B-IT' = 33.8. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "qwen2-7b",
    "bench": "gpqa-unspecified",
    "score": 0.384,
    "source": "https://github.com/InternLM/InternLM/blob/68fdc71330059255fd647abf28219ad11e8e4865/model_cards/internlm2.5_7b.md",
    "by": "independent",
    "date": "2024-07-03",
    "note": "InternLM GitHub model card for InternLM2.5-7B (model_cards/internlm2.5_7b.md, Apache-2.0 repository), chat-model table, row 'GPQA (0-shot)', column 'Qwen2-7B-Instruct' = 38.4. The card: 'We have evaluated InternLM2.5 on several important benchmarks using the open-source evaluation tool OpenCompass' and 'The evaluation results were obtained from OpenCompass'. Yi-1.5-9B-Chat and GLM-4-9B-Chat columns not used (identical BBH and MATH values in both columns suggest a transcription error); the asterisked Gemma2 BBH cell not used. MATH split and GPQA subset not stated. Date is the InternLM2.5 release (2024-07-03, model zoo table).",
    "source_id": "internlm-github-model-cards"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "mmlu",
    "score": 0.545,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'google/flan-t5-xxl', column 'MMLU' = 54.5; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "bbh",
    "score": 0.439,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'google/flan-t5-xxl', column 'BBH' = 43.9; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "flan-t5-xl",
    "bench": "mmlu",
    "score": 0.492,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'google/flan-t5-xl', column 'MMLU' = 49.2; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "flan-t5-xl",
    "bench": "bbh",
    "score": 0.402,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'google/flan-t5-xl', column 'BBH' = 40.2; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "koala-13b",
    "bench": "mmlu",
    "score": 0.446,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'TheBloke/koala-13B-HF', column 'MMLU' = 44.6; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). TheBloke/koala-13B-HF is a Transformers-format merge of the Koala 13B release (young-geng/koala delta weights on LLaMA 13B). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "koala-13b",
    "bench": "bbh",
    "score": 0.346,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'TheBloke/koala-13B-HF', column 'BBH' = 34.6; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). TheBloke/koala-13B-HF is a Transformers-format merge of the Koala 13B release (young-geng/koala delta weights on LLaMA 13B). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "koala-13b",
    "bench": "humaneval",
    "score": 0.11,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'TheBloke/koala-13B-HF', column 'HumanEval' = 11.0; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). TheBloke/koala-13B-HF is a Transformers-format merge of the Koala 13B release (young-geng/koala delta weights on LLaMA 13B). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "chatglm-6b",
    "bench": "mmlu",
    "score": 0.361,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'THUDM/chatglm-6b', column 'MMLU' = 36.1; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "chatglm-6b",
    "bench": "bbh",
    "score": 0.313,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'THUDM/chatglm-6b', column 'BBH' = 31.3; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "chatglm-6b",
    "bench": "humaneval",
    "score": 0.031,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'THUDM/chatglm-6b', column 'HumanEval' = 3.1; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-13b",
    "bench": "mmlu",
    "score": 0.462,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-13b-hf', column 'MMLU' = 46.2; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 13B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-13b",
    "bench": "bbh",
    "score": 0.371,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-13b-hf', column 'BBH' = 37.1; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 13B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-13b",
    "bench": "humaneval",
    "score": 0.134,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-13b-hf', column 'HumanEval' = 13.4; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 13B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-7b",
    "bench": "mmlu",
    "score": 0.352,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-7b-hf', column 'MMLU' = 35.2; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 7B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-7b",
    "bench": "bbh",
    "score": 0.309,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-7b-hf', column 'BBH' = 30.9; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 7B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "llama-7b",
    "bench": "humaneval",
    "score": 0.103,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'decapoda-research/llama-7b-hf', column 'HumanEval' = 10.3; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). Community HF conversion of the original LLaMA 7B weights. DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "mmlu",
    "score": 0.27,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5', column 'MMLU' = 27.0; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "bbh",
    "score": 0.3,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5', column 'BBH' = 30.0; BBH (3-shot, 6,511 questions); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "humaneval",
    "score": 0.091,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5', column 'HumanEval' = 9.1; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "mmlu",
    "score": 0.262,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'stabilityai/stablelm-base-alpha-7b', column 'MMLU' = 26.2; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "humaneval",
    "score": 0.018,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'stabilityai/stablelm-base-alpha-7b', column 'HumanEval' = 1.8; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "dolly-v2-12b",
    "bench": "mmlu",
    "score": 0.257,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'databricks/dolly-v2-12b', column 'MMLU' = 25.7; MMLU (5-shot, 14,042 questions per the paper); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "dolly-v2-12b",
    "bench": "humaneval",
    "score": 0.079,
    "source": "https://github.com/declare-lab/instruct-eval/blob/49bbd2ce7e3a5bdc4024ba62a5c9c68c107ee966/README.md",
    "by": "independent",
    "date": "2024-03-10",
    "note": "InstructEval (DeCLaRe Lab) README results table @49bbd2c, row model path 'databricks/dolly-v2-12b', column 'HumanEval' = 7.9; HumanEval (0-shot pass@1, 164 problems); run by the InstructEval authors with their open-source harness (paper arXiv:2306.04757, CC BY-SA 4.0, describes the setup). DROP column not used (metric not stated). GPT-4/ChatGPT rows (copied from OpenAI's report) not used. Repository licences: LICENSE (MIT) and LICENCE (Apache-2.0). Date is the README commit date.",
    "source_id": "instruct-eval-readme"
  },
  {
    "model": "hy3",
    "bench": "swe-bench-multilingual",
    "score": 0.758,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'SWE-bench Multilingual' printed '75.8' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "swe-bench-multilingual",
    "score": 0.829,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'SWE-bench Multilingual' printed '82.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.579,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'SWE-bench Pro' printed '57.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.657,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'SWE-bench Pro' printed '65.7' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "terminal-bench-2.1",
    "score": 0.708,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'Terminal-Bench 2.1' printed '70.8' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "terminal-bench-2.1",
    "score": 0.854,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'Terminal-Bench 2.1' printed '85.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "nl2repo",
    "score": 0.456,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'NL2Repo-Bench' printed '45.6' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "nl2repo",
    "score": 0.589,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'NL2Repo-Bench' printed '58.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "widesearch",
    "score": 0.819,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'WideSearch' printed '81.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "widesearch",
    "score": 0.839,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'WideSearch' printed '83.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "mcp-atlas",
    "score": 0.75,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'MCP-Atlas (public)' printed '75.0' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "mcp-atlas",
    "score": 0.837,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'MCP-Atlas (public)' printed '83.7' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "apex-agents",
    "score": 0.244,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'APEX-Agents (pass@1)' printed '24.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "apex-agents",
    "score": 0.371,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'APEX-Agents (pass@1)' printed '37.1' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "skillsbench-79-text-claude-code",
    "score": 0.553,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'SkillsBench (79, text-only)' printed '55.3' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "skillsbench-79-text-claude-code",
    "score": 0.629,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'SkillsBench (79, text-only)' printed '62.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "hle-with-tools",
    "score": 0.519,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'HLE (with tools, text-only)' printed '51.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "hle-with-tools",
    "score": 0.554,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'HLE (with tools, text-only)' printed '55.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "gpqa-diamond",
    "score": 0.909,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'GPQA Diamond' printed '90.9' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "gpqa-diamond",
    "score": 0.923,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'GPQA Diamond' printed '92.3' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "hle",
    "score": 0.344,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'HLE (no tools, text-only)' printed '34.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "hle",
    "score": 0.434,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'HLE (no tools, text-only)' printed '43.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "superchem",
    "score": 0.526,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'SUPERChem' printed '52.6' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "superchem",
    "score": 0.664,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'SUPERChem' printed '66.4' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3",
    "bench": "matharena-apex",
    "score": 0.387,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy3', row 'MathArena Apex 2025' printed '38.7' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy4-preview",
    "bench": "matharena-apex",
    "score": 0.742,
    "source": "https://huggingface.co/tencent/Hy4-preview/blob/705d81ee51566a186d645b74c974d642ef2828fe/README.md#benchmark-appendix",
    "by": "lab",
    "date": "2026-08-28",
    "note": "Tencent Hy own results in the Benchmark Appendix table image committed in the pinned Apache-2.0 tencent/Hy4-preview repository (assets/benchmark-appendix.jpg, sha256 02c751bf51c52c52cc02094561d7144bbfe3e27084d48d8a405d580da3554e21, embedded in the README 'Benchmark Appendix'; visually read). Column 'Hy4 preview', row 'MathArena Apex 2025' printed '74.2' (no asterisk: Tencent's own model). Appendix notes: each model at its highest available reasoning setting; SWE-bench Multilingual/Pro with the swe-agent scaffold; Terminal-Bench 2.1 with the Claude Code harness (up to 500 turns, 12-hour timeout, 16 CPU/32 GB); NL2Repo via Claude Code (1000-turn budget); MCP-Atlas with Scale's April 2026 methodology on the 500-task public set (100 tool calls/task, Gemini 3.1 Pro Preview judge); SkillsBench via Claude Code on 79 self-contained tasks, mean of 3 runs; APEX-Agents with the official ReAct Toolbelt harness (250 steps); agentic search with Tencent's internal harness. The appendix notes that some Hy3 scores differ from previously reported ones after harness/judge/anti-hacking updates. Internal Hy-* rows, DeepSWE, SWE Atlas, ArXivMath, BrokenArXiv, HorizonMath, CritPt, Toolathlon-Verified, ALE-CLI and GDPval Elo not used. Competitor columns (asterisked Tencent runs) not imported.",
    "source_id": "card2-tencent-hy4-preview"
  },
  {
    "model": "hy3-preview",
    "bench": "swe-bench-multilingual",
    "score": 0.683,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'SWE-bench Multilingual' printed '68.3'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "swe-bench-verified",
    "score": 0.744,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'SWE-bench Verified' printed '74.4'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.46,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'SWE-bench Pro' printed '46.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "terminal-bench-2.1",
    "score": 0.58,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'Terminal-Bench 2.1' printed '58.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "nl2repo",
    "score": 0.353,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'NL2repo' printed '35.3'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "browsecomp-context-managed",
    "score": 0.671,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'BrowseComp' printed '67.1'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "widesearch",
    "score": 0.678,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'WideSearch' printed '67.8'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "deepsearchqa",
    "score": 0.828,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'DeepSearchQA' printed '82.8'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "mcp-atlas",
    "score": 0.661,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'MCP atlas (public)' printed '66.1'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "toolathlon",
    "score": 0.321,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'Toolathlon' printed '32.1'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "apex-agents",
    "score": 0.124,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'Apex-Agent (pass@1)' printed '12.4'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "claw-eval-pass3",
    "score": 0.55,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'ClawEval (pass^3)' printed '55.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "wildclawbench-35-text",
    "score": 0.453,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'WildClawBench (35, text-only)' printed '45.3'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "skillsbench-79-text-claude-code",
    "score": 0.291,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'SkillsBench(79,text-only)' printed '29.1'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "hle-with-tools",
    "score": 0.354,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'HLE (with tools, text-only)' printed '35.4'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "gpqa-diamond",
    "score": 0.872,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'GPQA Diamond' printed '87.2'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "hle",
    "score": 0.3,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'HLE (no tools, text-only)' printed '30.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "frontierscience-research",
    "score": 0.19,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'FrontierScience-Research' printed '19.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "frontierscience-olympiad",
    "score": 0.7,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'FrontierScience-Olympiad' printed '70.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "usamo-2026",
    "score": 0.373,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'USAMO 2026' printed '37.3'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "matharena-apex",
    "score": 0.126,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'MathArena Apex' printed '12.6'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "phybench",
    "score": 0.714,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'PHYBench' printed '71.4'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "cmt-benchmark",
    "score": 0.194,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'CMT-Benchmark' printed '19.4'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "imo-answerbench",
    "score": 0.843,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'IMOAnswerBench' printed '84.3'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "superchem",
    "score": 0.47,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'SuperChem' printed '47.0'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "cl-bench",
    "score": 0.228,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'CL-bench' printed '22.8'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "hy3-preview",
    "bench": "cl-bench-life",
    "score": 0.157,
    "source": "https://huggingface.co/tencent/Hy3/blob/a960ebc3da325ba167f069f76c41eb62c9280d22/README.md",
    "by": "lab",
    "date": "2026-07-20",
    "note": "Tencent Hy own results in the benchmark appendix table image committed in the pinned tencent/Hy3 repository (assets/benchmark-appendix.png, sha256 c2b5092a0dd1207a6b356fed2047956bbce13e8441943c19b266ef9bbca3d174, embedded in the README 'Benchmark Appendix'; visually read). 'Hy3 preview' column (Tencent's own earlier checkpoint), row 'CL-bench life' printed '15.7'; same table, benchmark ids and appendix notes as the existing Hy3-column rows from this card (highest reasoning tier; SWE-agent scaffold for SWE-bench; Terminal-Bench 2.1 with Terminus-2; NL2Repo via Claude Code; SkillsBench via Claude Code on 79 text-only tasks, 3 runs; MCP-Atlas April 2026 methodology on the 500-task public set; agentic search via internal harness with self-summary context management for BrowseComp).",
    "source_id": "card2-tencent-hy3"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-verified",
    "score": 0.67,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'SWE-bench Verified w/ OpenHands, 100 turns' printed 67.0. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-live",
    "score": 0.263,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'SWE-bench Live' printed 26.3. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "swe-bench-multilingual",
    "score": 0.547,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'SWE-bench Multilingual' printed 54.7. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "aider-polyglot",
    "score": 0.618,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'Aider-Polyglot' printed 61.8. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "bfcl-v3",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'BFCL-v3' printed 68.7. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "tau-bench-retail",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'TAU-Bench Retail' printed 77.5. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen3-coder-480b-a35b",
    "bench": "tau-bench-airline",
    "score": 0.6,
    "source": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own grey 'Qwen3-Coder 480B-A35B-Instruct' column (Open Models section) in the chart embedded in the pinned Apache-2.0 Qwen3-Coder-30B-A3B-Instruct card, same chart and rows as the existing Qwen3-Coder-30B-A3B rows; row 'TAU-Bench Airline' printed 60.0. SWE-bench Verified uses the OpenHands 100-turn row (matching the 30B row; the 500-turn 69.6 not added). Kimi-K2, DeepSeek-V3, Claude and GPT columns not imported. External image URL content-hashed (sha256 fced99c42731e1371b7e692838354d149dcefd2f79ebe02fd4f86f7186bffbbf). Review date is not the evaluation-run date.",
    "source_id": "card-qwen-qwen3-coder-30b-a3b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mmmu-pro",
    "score": 0.511,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMMU Pro', column 'Qwen2.5-VL-72B' = 51.1 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmmu-pro",
    "score": 0.462,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMMU Pro', column 'Qwen2-VL-72B' = 46.2 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "mmmu-pro",
    "score": 0.495,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMMU Pro', column 'Qwen2.5-VL-32B' = 49.5 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mmstar",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMStar', column 'Qwen2.5-VL-72B' = 70.8 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmstar",
    "score": 0.683,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMStar', column 'Qwen2-VL-72B' = 68.3 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "mmstar",
    "score": 0.695,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'MMStar', column 'Qwen2.5-VL-32B' = 69.5 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "cc-ocr",
    "score": 0.798,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'CC-OCR', column 'Qwen2.5-VL-72B' = 79.8 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "cc-ocr",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'CC-OCR', column 'Qwen2-VL-72B' = 68.7 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "cc-ocr",
    "score": 0.771,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'CC-OCR', column 'Qwen2.5-VL-32B' = 77.1 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "docvqa",
    "score": 0.964,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'DocVQA', column 'Qwen2.5-VL-72B' = 96.4 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "docvqa",
    "score": 0.965,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'DocVQA', column 'Qwen2-VL-72B' = 96.5 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "docvqa",
    "score": 0.948,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'DocVQA', column 'Qwen2.5-VL-32B' = 94.8 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "lvbench",
    "score": 0.473,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'LVBench', column 'Qwen2.5-VL-72B' = 47.3 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "lvbench",
    "score": 0.49,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'LVBench', column 'Qwen2.5-VL-32B' = 49.00 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "videomme-without-sub",
    "score": 0.733,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2.5-VL-72B' = 73.3/79.1 (all three columns are Qwen's own models). Cell '73.3/79.1' is 'without subtitles/with subtitles'; the first value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "videomme-without-sub",
    "score": 0.712,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2-VL-72B' = 71.2/77.8 (all three columns are Qwen's own models). Cell '71.2/77.8' is 'without subtitles/with subtitles'; the first value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "videomme-without-sub",
    "score": 0.705,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2.5-VL-32B' = 70.5/77.9 (all three columns are Qwen's own models). Cell '70.5/77.9' is 'without subtitles/with subtitles'; the first value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "videomme-with-sub",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2.5-VL-72B' = 73.3/79.1 (all three columns are Qwen's own models). Cell '73.3/79.1' is 'without subtitles/with subtitles'; the second value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "videomme-with-sub",
    "score": 0.778,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2-VL-72B' = 71.2/77.8 (all three columns are Qwen's own models). Cell '71.2/77.8' is 'without subtitles/with subtitles'; the second value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "videomme-with-sub",
    "score": 0.779,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'VideoMME', column 'Qwen2.5-VL-32B' = 70.5/77.9 (all three columns are Qwen's own models). Cell '70.5/77.9' is 'without subtitles/with subtitles'; the second value is used. MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "screenspot-pro",
    "score": 0.436,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'ScreenSpot Pro', column 'Qwen2.5-VL-72B' = 43.6 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "screenspot-pro",
    "score": 0.394,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'ScreenSpot Pro', column 'Qwen2.5-VL-32B' = 39.4 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "androidworld",
    "score": 0.35,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'AndroidWorld', column 'Qwen2.5-VL-72B' = 35 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "androidworld",
    "score": 0.22,
    "source": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct/blob/7cfb30d71a1f4f49a57592323337a4a4727301da/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-21",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2.5-VL-32B-Instruct model card, Evaluation / Vision table, row 'AndroidWorld', column 'Qwen2.5-VL-32B' = 22.0 (all three columns are Qwen's own models). MMMU and MathVista rows not used (split not printed); OCRBenchV2, InfoVQA, MathVision, AITZ, Android Control, ScreenSpot (v1), MMBench-Video, CharadesSTA and OSWorld (not Verified) have no exact catalogue id. Text table not used (competitor columns not marked as Qwen-run). Date is the model repository creation date (2025-03-21).",
    "source_id": "card2-qwen-qwen2-5-vl-32b-instruct"
  },
  {
    "model": "glm-4-6v",
    "bench": "mmstar",
    "score": 0.759,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMStar', column 'GLM-4.6V (106B-A12B)' = 75.9. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "mmstar",
    "score": 0.753,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMStar', column 'GLM-4.5V (106B-A12B)' = 75.3. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "mmmu",
    "score": 0.76,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMMU (Val)', column 'GLM-4.6V (106B-A12B)' = 76.0. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "mmmu",
    "score": 0.754,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMMU (Val)', column 'GLM-4.5V (106B-A12B)' = 75.4. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "mmmu-pro",
    "score": 0.66,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMMU_Pro', column 'GLM-4.6V (106B-A12B)' = 66.0. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "mmmu-pro",
    "score": 0.652,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'MMMU_Pro', column 'GLM-4.5V (106B-A12B)' = 65.2. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "androidworld",
    "score": 0.57,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'AndroidWorld', column 'GLM-4.6V (106B-A12B)' = 57.0. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "androidworld",
    "score": 0.57,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'AndroidWorld', column 'GLM-4.5V (106B-A12B)' = 57.0. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "lvbench",
    "score": 0.595,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'LVBench', column 'GLM-4.6V (106B-A12B)' = 59.5. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "lvbench",
    "score": 0.538,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'LVBench', column 'GLM-4.5V (106B-A12B)' = 53.8. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "ocrbench",
    "score": 0.865,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'OCRBench', column 'GLM-4.6V (106B-A12B)' = 86.5. GLM-4.6V and GLM-4.5V are both Z.ai's own models. OCRBench printed on a 0-100 scale (raw 0-1000 divided by 10), as the catalogue id expects. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "ocrbench",
    "score": 0.865,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'OCRBench', column 'GLM-4.5V (106B-A12B)' = 86.5. GLM-4.6V and GLM-4.5V are both Z.ai's own models. OCRBench printed on a 0-100 scale (raw 0-1000 divided by 10), as the catalogue id expects. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-6v",
    "bench": "charxiv-rq",
    "score": 0.632,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'CharXiv_Val-Reasoning', column 'GLM-4.6V (106B-A12B)' = 63.2. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "glm-4-5v",
    "bench": "charxiv-rq",
    "score": 0.584,
    "source": "https://github.com/zai-org/GLM-V/blob/9013a771cfe578d8a0c7dfc4548156f3a23e4890/resources/bench_46v.jpeg",
    "by": "lab",
    "date": "2025-12-08",
    "note": "Z.ai's benchmark table image for GLM-4.6V (resources/bench_46v.jpeg in the zai-org/GLM-V GitHub repository @9013a77, Apache-2.0; also embedded in the MIT GLM-4.6V Hugging Face card), visually read: row 'CharXiv_Val-Reasoning', column 'GLM-4.5V (106B-A12B)' = 58.4. GLM-4.6V and GLM-4.5V are both Z.ai's own models. Other benchmarks without an exact catalogue id (MMBench split, MathVista/AI2D split, OSWorld non-Verified, etc.) and the other labs' columns not used. Date is the image's commit date. Image sha256 8ce205c218a2a9815698117c662b3fda8bbb48f33fb966d588232be0ef7fae7e.",
    "source_id": "glm-v-github-bench-image"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmlu-pro",
    "score": 0.818,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-Pro', column 'Qwen3-VL-235B-A22B-Instruct' = 81.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.83,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-Pro', column 'Qwen3-235B-A22B-Instruct-2507' = 83.0. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmlu-redux",
    "score": 0.922,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-Redux', column 'Qwen3-VL-235B-A22B-Instruct' = 92.2. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.931,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-Redux', column 'Qwen3-235B-A22B-Instruct-2507' = 93.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "supergpqa",
    "score": 0.604,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'SuperGPQA', column 'Qwen3-VL-235B-A22B-Instruct' = 60.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.626,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'SuperGPQA', column 'Qwen3-235B-A22B-Instruct-2507' = 62.6. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "simpleqa",
    "score": 0.519,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'SimpleQA', column 'Qwen3-VL-235B-A22B-Instruct' = 51.9. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "simpleqa",
    "score": 0.543,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'SimpleQA', column 'Qwen3-235B-A22B-Instruct-2507' = 54.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "chinese-simpleqa",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'CSimpleQA', column 'Qwen3-VL-235B-A22B-Instruct' = 83.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "chinese-simpleqa",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'CSimpleQA', column 'Qwen3-235B-A22B-Instruct-2507' = 84.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "aime-2025",
    "score": 0.747,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'AIME25', column 'Qwen3-VL-235B-A22B-Instruct' = 74.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'AIME25', column 'Qwen3-235B-A22B-Instruct-2507' = 70.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'HMMT25', column 'Qwen3-VL-235B-A22B-Instruct' = 57.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.554,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'HMMT25', column 'Qwen3-235B-A22B-Instruct-2507' = 55.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "livebench-20241125",
    "score": 0.748,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'LiveBench1125', column 'Qwen3-VL-235B-A22B-Instruct' = 74.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livebench-20241125",
    "score": 0.754,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'LiveBench1125', column 'Qwen3-235B-A22B-Instruct-2507' = 75.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "livecodebench-v6",
    "score": 0.543,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'LCBV6 (25.02 - 25.05)', column 'Qwen3-VL-235B-A22B-Instruct' = 54.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'LCBV6 (25.02 - 25.05)', column 'Qwen3-235B-A22B-Instruct-2507' = 51.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "multipl-e",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MultiPL-E', column 'Qwen3-VL-235B-A22B-Instruct' = 86.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "multipl-e",
    "score": 0.879,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MultiPL-E', column 'Qwen3-235B-A22B-Instruct-2507' = 87.9. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'IFEval', column 'Qwen3-VL-235B-A22B-Instruct' = 87.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.887,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'IFEval', column 'Qwen3-235B-A22B-Instruct-2507' = 88.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "arena-hard-v2",
    "score": 0.774,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'Arena-Hard v2', column 'Qwen3-VL-235B-A22B-Instruct' = 77.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "arena-hard-v2",
    "score": 0.792,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'Arena-Hard v2', column 'Qwen3-235B-A22B-Instruct-2507' = 79.2. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "creative-writing-v3",
    "score": 0.865,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'Creative Writing v3', column 'Qwen3-VL-235B-A22B-Instruct' = 86.5. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "creative-writing-v3",
    "score": 0.875,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'Creative Writing v3', column 'Qwen3-235B-A22B-Instruct-2507' = 87.5. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "writingbench",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'WritingBench', column 'Qwen3-VL-235B-A22B-Instruct' = 85.5. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "writingbench",
    "score": 0.852,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'WritingBench', column 'Qwen3-235B-A22B-Instruct-2507' = 85.2. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "bfcl-v3",
    "score": 0.677,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'BFCL-v3', column 'Qwen3-VL-235B-A22B-Instruct' = 67.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.709,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'BFCL-v3', column 'Qwen3-235B-A22B-Instruct-2507' = 70.9. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "multi-if-unspecified",
    "score": 0.763,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MultiIF', column 'Qwen3-VL-235B-A22B-Instruct' = 76.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "multi-if-unspecified",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MultiIF', column 'Qwen3-235B-A22B-Instruct-2507' = 77.5. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmlu-prox",
    "score": 0.778,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-ProX', column 'Qwen3-VL-235B-A22B-Instruct' = 77.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-prox",
    "score": 0.794,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'MMLU-ProX', column 'Qwen3-235B-A22B-Instruct-2507' = 79.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "include",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'INCLUDE', column 'Qwen3-VL-235B-A22B-Instruct' = 80.0. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "include",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct/blob/710c13861be6c466e66de3f484069440b8f31389/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_nothinking_text.jpg, sha256 21892e4214d200cc471c144f6548a58cf2d9e7e5a3e387ad382899666984739f), visually read: row 'INCLUDE', column 'Qwen3-235B-A22B-Instruct-2507' = 79.5. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Instruct-2507 values agree with that model's own card. Arena-Hard v2 win rates judged by GPT-4.1 (table footnote). Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmlu-pro",
    "score": 0.838,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-Pro', column 'Qwen3-VL-235B-A22B-Thinking' = 83.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-Pro', column 'Qwen3-235B-A22B-Thinking-2507' = 84.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmlu-redux",
    "score": 0.937,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-Redux', column 'Qwen3-VL-235B-A22B-Thinking' = 93.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.938,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-Redux', column 'Qwen3-235B-A22B-Thinking-2507' = 93.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "supergpqa",
    "score": 0.643,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'SuperGPQA', column 'Qwen3-VL-235B-A22B-Thinking' = 64.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'SuperGPQA', column 'Qwen3-235B-A22B-Thinking-2507' = 64.9. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "aime-2025",
    "score": 0.897,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'AIME25', column 'Qwen3-VL-235B-A22B-Thinking' = 89.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.923,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'AIME25', column 'Qwen3-235B-A22B-Thinking-2507' = 92.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.774,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'HMMT25', column 'Qwen3-VL-235B-A22B-Thinking' = 77.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'HMMT25', column 'Qwen3-235B-A22B-Thinking-2507' = 83.9. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "livecodebench-v6",
    "score": 0.701,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'LCB v6', column 'Qwen3-VL-235B-A22B-Thinking' = 70.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.741,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'LCB v6', column 'Qwen3-235B-A22B-Thinking-2507' = 74.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "livebench-20241125",
    "score": 0.796,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'LiveBench1125', column 'Qwen3-VL-235B-A22B-Thinking' = 79.6. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'LiveBench1125', column 'Qwen3-235B-A22B-Thinking-2507' = 78.4. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "hle",
    "score": 0.136,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'HLE', column 'Qwen3-VL-235B-A22B-Thinking' = 13.6. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hle",
    "score": 0.182,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'HLE', column 'Qwen3-235B-A22B-Thinking-2507' = 18.2. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.882,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'IFEval', column 'Qwen3-VL-235B-A22B-Thinking' = 88.2. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'IFEval', column 'Qwen3-235B-A22B-Thinking-2507' = 87.8. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "creative-writing-v3",
    "score": 0.857,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'Creative Writing v3', column 'Qwen3-VL-235B-A22B-Thinking' = 85.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "creative-writing-v3",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'Creative Writing v3', column 'Qwen3-235B-A22B-Thinking-2507' = 86.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "writingbench",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'WritingBench', column 'Qwen3-VL-235B-A22B-Thinking' = 86.7. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "writingbench",
    "score": 0.883,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'WritingBench', column 'Qwen3-235B-A22B-Thinking-2507' = 88.3. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "multi-if-unspecified",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MultiIF', column 'Qwen3-VL-235B-A22B-Thinking' = 79.1. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MultiIF', column 'Qwen3-235B-A22B-Thinking-2507' = 80.6. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmlu-prox",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-ProX', column 'Qwen3-VL-235B-A22B-Thinking' = 80.6. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'MMLU-ProX', column 'Qwen3-235B-A22B-Thinking-2507' = 81.0. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "include",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'INCLUDE', column 'Qwen3-VL-235B-A22B-Thinking' = 80.0. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "include",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking/blob/6664affde68449468deb7527186455c7450c13c0/README.md#model-performance",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Qwen's own text-benchmark table image embedded in the pinned Apache-2.0 card (https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/table_thinking_text.jpg, sha256 4cce76a4c25eaef1d5ffcc6c2591140a7ad105a6ce7e634636f340dfaafecd75), visually read: row 'INCLUDE', column 'Qwen3-235B-A22B-Thinking-2507' = 81.0. Both compared Qwen columns are Qwen's own models; the Qwen3-235B-A22B-Thinking-2507 values agree with that model's own card. BFCL row not used (the two Qwen values look swapped relative to the Thinking-2507 card); SimpleQA and ZebraLogic rows not used. Other-lab columns (DeepSeek, Claude, OpenAI) and rows without an exact catalogue id (MMLU, LCBV5, SIFO) not used. External image URL is content-hashed and tied to the pinned README. Date is the model repository creation date (2025-09-22).",
    "source_id": "card2-qwen-qwen3-vl-235b-a22b-thinking"
  },
  {
    "model": "gemma-2-2b",
    "bench": "ifeval-unspecified",
    "score": 0.5383,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'IFEval' (0-shot), column 'Gemma-2 2B' = 53.83. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "ifeval-unspecified",
    "score": 0.51,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'IFEval' (0-shot), column 'Llama-3.2 3B' = 51.00. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "ifeval-unspecified",
    "score": 0.4607,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'IFEval' (0-shot), column 'Granite-3.0 2B' = 46.07. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "ifeval-unspecified",
    "score": 0.5037,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'IFEval' (0-shot), column 'Llama-3.1 8B' = 50.37. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "ifeval-unspecified",
    "score": 0.5227,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'IFEval' (0-shot), column 'Granite-3.0 8B' = 52.27. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "mmlu",
    "score": 0.5683,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU' (5-shot), column 'Gemma-2 2B' = 56.83. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu",
    "score": 0.5968,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU' (5-shot), column 'Llama-3.2 3B' = 59.68. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "mmlu",
    "score": 0.5603,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU' (5-shot), column 'Granite-3.0 2B' = 56.03. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu",
    "score": 0.6827,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU' (5-shot), column 'Llama-3.1 8B' = 68.27. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "mmlu",
    "score": 0.6582,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU' (5-shot), column 'Granite-3.0 8B' = 65.82. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "mmlu-pro",
    "score": 0.2719,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU-Pro' (5-shot), column 'Gemma-2 2B' = 27.19. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu-pro",
    "score": 0.3006,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU-Pro' (5-shot), column 'Llama-3.2 3B' = 30.06. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "mmlu-pro",
    "score": 0.2792,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU-Pro' (5-shot), column 'Granite-3.0 2B' = 27.92. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu-pro",
    "score": 0.3797,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU-Pro' (5-shot), column 'Llama-3.1 8B' = 37.97. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "mmlu-pro",
    "score": 0.3445,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MMLU-Pro' (5-shot), column 'Granite-3.0 8B' = 34.45. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "openbookqa",
    "score": 0.442,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'OBQA' (0-shot), column 'Gemma-2 2B' = 44.20. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "openbookqa",
    "score": 0.36,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'OBQA' (0-shot), column 'Llama-3.2 3B' = 36.00. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "openbookqa",
    "score": 0.432,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'OBQA' (0-shot), column 'Granite-3.0 2B' = 43.20. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "openbookqa",
    "score": 0.43,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'OBQA' (0-shot), column 'Llama-3.1 8B' = 43.00. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "openbookqa",
    "score": 0.466,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'OBQA' (0-shot), column 'Granite-3.0 8B' = 46.60. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "social-iqa",
    "score": 0.6083,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'SIQA' (0-shot), column 'Gemma-2 2B' = 60.83. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "social-iqa",
    "score": 0.5798,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'SIQA' (0-shot), column 'Llama-3.2 3B' = 57.98. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "social-iqa",
    "score": 0.6636,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'SIQA' (0-shot), column 'Granite-3.0 2B' = 66.36. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "social-iqa",
    "score": 0.6501,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'SIQA' (0-shot), column 'Llama-3.1 8B' = 65.01. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "social-iqa",
    "score": 0.7121,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'SIQA' (0-shot), column 'Granite-3.0 8B' = 71.21. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "hellaswag",
    "score": 0.7121,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'Hellaswag' (10-shot), column 'Gemma-2 2B' = 71.21. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "hellaswag",
    "score": 0.7347,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'Hellaswag' (10-shot), column 'Llama-3.2 3B' = 73.47. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "hellaswag",
    "score": 0.7679,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'Hellaswag' (10-shot), column 'Granite-3.0 2B' = 76.79. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "hellaswag",
    "score": 0.8012,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'Hellaswag' (10-shot), column 'Llama-3.1 8B' = 80.12. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "hellaswag",
    "score": 0.8261,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'Hellaswag' (10-shot), column 'Granite-3.0 8B' = 82.61. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "winogrande",
    "score": 0.689,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'WinoGrande' (5-shot), column 'Gemma-2 2B' = 68.90. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "winogrande",
    "score": 0.7017,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'WinoGrande' (5-shot), column 'Llama-3.2 3B' = 70.17. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "winogrande",
    "score": 0.719,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'WinoGrande' (5-shot), column 'Granite-3.0 2B' = 71.90. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "winogrande",
    "score": 0.7837,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'WinoGrande' (5-shot), column 'Llama-3.1 8B' = 78.37. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "winogrande",
    "score": 0.7751,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'WinoGrande' (5-shot), column 'Granite-3.0 8B' = 77.51. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "boolq",
    "score": 0.8437,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BoolQ' (5-shot), column 'Gemma-2 2B' = 84.37. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "boolq",
    "score": 0.8046,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BoolQ' (5-shot), column 'Llama-3.2 3B' = 80.46. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "boolq",
    "score": 0.8489,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BoolQ' (5-shot), column 'Granite-3.0 2B' = 84.89. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "boolq",
    "score": 0.8725,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BoolQ' (5-shot), column 'Llama-3.1 8B' = 87.25. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "boolq",
    "score": 0.8865,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BoolQ' (5-shot), column 'Granite-3.0 8B' = 88.65. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "arc-challenge",
    "score": 0.5742,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'ARC-C' (25-shot), column 'Gemma-2 2B' = 57.42. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "arc-challenge",
    "score": 0.3047,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'ARC-C' (25-shot), column 'Llama-3.2 3B' = 30.47. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "arc-challenge",
    "score": 0.5435,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'ARC-C' (25-shot), column 'Granite-3.0 2B' = 54.35. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "arc-challenge",
    "score": 0.6067,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'ARC-C' (25-shot), column 'Llama-3.1 8B' = 60.67. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "arc-challenge",
    "score": 0.6416,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'ARC-C' (25-shot), column 'Granite-3.0 8B' = 64.16. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "gpqa-unspecified",
    "score": 0.2936,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GPQA' (0-shot), column 'Gemma-2 2B' = 29.36. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gpqa-unspecified",
    "score": 0.2919,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GPQA' (0-shot), column 'Llama-3.2 3B' = 29.19. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "gpqa-unspecified",
    "score": 0.2861,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GPQA' (0-shot), column 'Granite-3.0 2B' = 28.61. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gpqa-unspecified",
    "score": 0.3213,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GPQA' (0-shot), column 'Llama-3.1 8B' = 32.13. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "gpqa-unspecified",
    "score": 0.3381,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GPQA' (0-shot), column 'Granite-3.0 8B' = 33.81. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "bbh",
    "score": 0.4348,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BBH' (3-shot), column 'Gemma-2 2B' = 43.48. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "bbh",
    "score": 0.4392,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BBH' (3-shot), column 'Llama-3.2 3B' = 43.92. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "bbh",
    "score": 0.4374,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BBH' (3-shot), column 'Granite-3.0 2B' = 43.74. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bbh",
    "score": 0.5081,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BBH' (3-shot), column 'Llama-3.1 8B' = 50.81. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "bbh",
    "score": 0.5155,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'BBH' (3-shot), column 'Granite-3.0 8B' = 51.55. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "humaneval",
    "score": 0.4055,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'HumanEvalSynthesis' (pass@1 (the paper: HumanEvalSynthesize refers to standard HumanEval)), column 'Gemma-2 2B' = 40.55. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "humaneval",
    "score": 0.4512,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'HumanEvalSynthesis' (pass@1 (the paper: HumanEvalSynthesize refers to standard HumanEval)), column 'Llama-3.2 3B' = 45.12. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "humaneval",
    "score": 0.5061,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'HumanEvalSynthesis' (pass@1 (the paper: HumanEvalSynthesize refers to standard HumanEval)), column 'Granite-3.0 2B' = 50.61. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "humaneval",
    "score": 0.6341,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'HumanEvalSynthesis' (pass@1 (the paper: HumanEvalSynthesize refers to standard HumanEval)), column 'Llama-3.1 8B' = 63.41. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "humaneval",
    "score": 0.6463,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'HumanEvalSynthesis' (pass@1 (the paper: HumanEvalSynthesize refers to standard HumanEval)), column 'Granite-3.0 8B' = 64.63. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "mbpp",
    "score": 0.34,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MBPP' (pass@1), column 'Gemma-2 2B' = 34.00. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mbpp",
    "score": 0.402,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MBPP' (pass@1), column 'Llama-3.2 3B' = 40.20. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "mbpp",
    "score": 0.41,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MBPP' (pass@1), column 'Granite-3.0 2B' = 41.00. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mbpp",
    "score": 0.522,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MBPP' (pass@1), column 'Llama-3.1 8B' = 52.20. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "mbpp",
    "score": 0.496,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MBPP' (pass@1), column 'Granite-3.0 8B' = 49.60. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "gsm8k",
    "score": 0.3086,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GSM8k' (5-shot CoT), column 'Gemma-2 2B' = 30.86. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gsm8k",
    "score": 0.5845,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GSM8k' (5-shot CoT), column 'Llama-3.2 3B' = 58.45. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "gsm8k",
    "score": 0.5966,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GSM8k' (5-shot CoT), column 'Granite-3.0 2B' = 59.66. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gsm8k",
    "score": 0.6504,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GSM8k' (5-shot CoT), column 'Llama-3.1 8B' = 65.04. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "gsm8k",
    "score": 0.6899,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'GSM8k' (5-shot CoT), column 'Granite-3.0 8B' = 68.99. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "gemma-2-2b",
    "bench": "math-subset-unspecified",
    "score": 0.2176,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MATH' (4-shot), column 'Gemma-2 2B' = 21.76. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "math-subset-unspecified",
    "score": 0.3136,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MATH' (4-shot), column 'Llama-3.2 3B' = 31.36. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "math-subset-unspecified",
    "score": 0.2366,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MATH' (4-shot), column 'Granite-3.0 2B' = 23.66. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "math-subset-unspecified",
    "score": 0.3446,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MATH' (4-shot), column 'Llama-3.1 8B' = 34.46. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "math-subset-unspecified",
    "score": 0.3094,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 10 'Instruct version performance of Granite-3.0 dense and baseline models', row 'MATH' (4-shot), column 'Granite-3.0 8B' = 30.94. Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'. All columns are instruct models. Mistral 7B column not used (version not stated); AGI-Eval, TruthfulQA, SQuAD 2.0, HumanEvalExplain/Fix, PAWS-X, MGSM (6 langs) and MT-Bench rows have no exact catalogue id. Date is the last commit of the pinned paper.pdf (2024-11-09; first uploaded 2024-10-21).",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "mmlu-redux",
    "score": 0.923,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'MMLU-Redux', column 'EXAONE 4.0 32B' printed 92.3. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "mmlu-pro",
    "score": 0.818,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'MMLU-Pro', column 'EXAONE 4.0 32B' printed 81.8. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "gpqa-diamond",
    "score": 0.754,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'GPQA-Diamond', column 'EXAONE 4.0 32B' printed 75.4. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "aime-2025",
    "score": 0.853,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'AIME 2025', column 'EXAONE 4.0 32B' printed 85.3. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "hmmt-feb-2025",
    "score": 0.729,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'HMMT Feb 2025', column 'EXAONE 4.0 32B' printed 72.9. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "ifeval-unspecified",
    "score": 0.837,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'IFEval', column 'EXAONE 4.0 32B' printed 83.7. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "bfcl-v3",
    "score": 0.639,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'BFCL-v3', column 'EXAONE 4.0 32B' printed 63.9. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau-bench-airline",
    "score": 0.515,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'Tau-Bench (Airline)', column 'EXAONE 4.0 32B' printed 51.5. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau-bench-retail",
    "score": 0.628,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '32B Reasoning Mode', row 'Tau-Bench (Retail)', column 'EXAONE 4.0 32B' printed 62.8. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "mmlu-redux",
    "score": 0.715,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'MMLU-Redux', column 'EXAONE 4.0 1.2B' printed 71.5. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "mmlu-pro",
    "score": 0.593,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'MMLU-Pro', column 'EXAONE 4.0 1.2B' printed 59.3. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "gpqa-diamond",
    "score": 0.52,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'GPQA-Diamond', column 'EXAONE 4.0 1.2B' printed 52.0. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "aime-2025",
    "score": 0.452,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'AIME 2025', column 'EXAONE 4.0 1.2B' printed 45.2. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "hmmt-feb-2025",
    "score": 0.34,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'HMMT Feb 2025', column 'EXAONE 4.0 1.2B' printed 34.0. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "ifeval-unspecified",
    "score": 0.678,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'IFEval', column 'EXAONE 4.0 1.2B' printed 67.8. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "bfcl-v3",
    "score": 0.529,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'BFCL-v3', column 'EXAONE 4.0 1.2B' printed 52.9. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "tau-bench-airline",
    "score": 0.205,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'Tau-Bench (Airline)', column 'EXAONE 4.0 1.2B' printed 20.5. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-1-2b",
    "bench": "tau-bench-retail",
    "score": 0.281,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B/blob/a1d54d1c148c30881ed27e035b650da489b51b92/README.md#performance",
    "by": "lab",
    "date": "2025-07-11",
    "note": "LG AI Research EXAONE-4.0-32B model card (pinned a1d54d1; the 1.2B card carries the same tables), Performance section, table '1.2B Reasoning Mode', row 'Tau-Bench (Retail)', column 'EXAONE 4.0 1.2B' printed 28.1. Reasoning mode; evaluation details in LG's EXAONE 4.0 technical report. LiveCodeBench rows not used (window not printed), Multi-IF (EN) and Korean benchmarks have no exact catalogue id. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "aime-2025",
    "score": 0.929,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-04-04",
    "note": "LG AI Research EXAONE-4.5-33B model card (pinned 570aa4b), Evaluation Results / Language-only Tasks table, row 'AIME 2025', column 'EXAONE 4.5 33B (Reasoning)' printed 92.9. Only benchmarks also reported for EXAONE 4.0 on LG's cards are recorded; tau2-bench, IFBench, AIME 2026 and vision rows would form single-model runs and are left out. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "gpqa-diamond",
    "score": 0.805,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-04-04",
    "note": "LG AI Research EXAONE-4.5-33B model card (pinned 570aa4b), Evaluation Results / Language-only Tasks table, row 'GPQA-Diamond', column 'EXAONE 4.5 33B (Reasoning)' printed 80.5. Only benchmarks also reported for EXAONE 4.0 on LG's cards are recorded; tau2-bench, IFBench, AIME 2026 and vision rows would form single-model runs and are left out. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "mmlu-pro",
    "score": 0.833,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-04-04",
    "note": "LG AI Research EXAONE-4.5-33B model card (pinned 570aa4b), Evaluation Results / Language-only Tasks table, row 'MMLU-Pro', column 'EXAONE 4.5 33B (Reasoning)' printed 83.3. Only benchmarks also reported for EXAONE 4.0 on LG's cards are recorded; tau2-bench, IFBench, AIME 2026 and vision rows would form single-model runs and are left out. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "ifeval-unspecified",
    "score": 0.896,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-04-04",
    "note": "LG AI Research EXAONE-4.5-33B model card (pinned 570aa4b), Evaluation Results / Language-only Tasks table, row 'IFEval', column 'EXAONE 4.5 33B (Reasoning)' printed 89.6. Only benchmarks also reported for EXAONE 4.0 on LG's cards are recorded; tau2-bench, IFBench, AIME 2026 and vision rows would form single-model runs and are left out. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "ifeval-unspecified",
    "score": 0.736,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct/blob/ccce25bd39c141fe053e0bc75818a8f5fe962802/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-2.4B-Instruct model card (pinned ccce25b), Evaluation table, row 'EXAONE 3.5 2.4B', column 'IFEval' printed 73.6 (aggregation not stated). MT-Bench, LiveBench (version not stated), Arena-Hard (version not stated), AlpacaEval, KoMT-Bench and LogicKor not used. Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "nova-premier",
    "bench": "mmlu",
    "score": 0.874,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'MMLU', column 'Nova Premier' printed 87.4; 0-shot CoT, macro-average exact match. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "mmlu",
    "score": 0.859,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'MMLU', column 'Nova Pro' printed 85.9; 0-shot CoT, macro-average exact match. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "gpqa-diamond",
    "score": 0.571,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'GPQA Diamond', column 'Nova Premier' printed 57.1; 0-shot CoT, exact match (±CI printed). Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "gpqa-diamond",
    "score": 0.5,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'GPQA Diamond', column 'Nova Pro' printed 50.0; 0-shot CoT, exact match (±CI printed). Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "aime-2025",
    "score": 0.16,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'AIME 2025', column 'Nova Premier' printed 16.0; mean exact match over 5 sampled responses per problem. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "aime-2025",
    "score": 0.053,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'AIME 2025', column 'Nova Pro' printed 5.3; mean exact match over 5 sampled responses per problem. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "math-500",
    "score": 0.82,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'MATH-500', column 'Nova Premier' printed 82.0; 0-shot CoT, exact match. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "math-500",
    "score": 0.766,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'MATH-500', column 'Nova Pro' printed 76.6; 0-shot CoT, exact match. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "bigcodebench",
    "score": 0.281,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'BigCodeBench Hard', column 'Nova Premier' printed 28.1; Hard split, pass@1. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "bigcodebench",
    "score": 0.223,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'BigCodeBench Hard', column 'Nova Pro' printed 22.3; Hard split, pass@1. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "charxiv-rq",
    "score": 0.488,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'CharXiv (Descriptive/Reasoning)', column 'Nova Premier' printed 48.8; reasoning value of the printed descriptive/reasoning pair (Premier 84.6/48.8, Pro 70.5/40.6), no CoT, GPT-4o-graded. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "charxiv-rq",
    "score": 0.406,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'CharXiv (Descriptive/Reasoning)', column 'Nova Pro' printed 40.6; reasoning value of the printed descriptive/reasoning pair (Premier 84.6/48.8, Pro 70.5/40.6), no CoT, GPT-4o-graded. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-premier",
    "bench": "bfcl-v3",
    "score": 0.637,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'BFCL (2025-04-25)', column 'Nova Premier' printed 63.7; BFCL leaderboard version of 2025-04-25 (the v3 benchmark set). Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "bfcl-v3",
    "score": 0.608,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1, row 'BFCL (2025-04-25)', column 'Nova Pro' printed 60.8; BFCL leaderboard version of 2025-04-25 (the v3 benchmark set). Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude and GPT-4.5 columns not used. MMMU (split not printed), IFEval (instruction-level loose, no exact id), MBXP, OCRBench-v2, EgoSchema, TallyQA, SimpleQA-with-SerpApi, ScreenSpot and SWE-bench (Premier only) not used. Prompt templates: huggingface.co/datasets/amazon-agi/Amazon-Nova-1.0-Premier-evals.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "trinity-large-thinking",
    "bench": "gpqa-diamond",
    "score": 0.763,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking/blob/dc6a99b6b74202880e31491a445a94884c97890c/README.md#benchmarks",
    "by": "lab",
    "date": "2026-04-01",
    "note": "Arcee AI Trinity-Large-Thinking model card (pinned), Benchmarks table, row 'GPQA-Diamond', column 'Trinity-Large-Thinking' printed 76.3. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "trinity-large-thinking",
    "bench": "aime-2025",
    "score": 0.963,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking/blob/dc6a99b6b74202880e31491a445a94884c97890c/README.md#benchmarks",
    "by": "lab",
    "date": "2026-04-01",
    "note": "Arcee AI Trinity-Large-Thinking model card (pinned), Benchmarks table, row 'AIME25', column 'Trinity-Large-Thinking' printed 96.3. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "trinity-large-thinking",
    "bench": "mmlu-pro",
    "score": 0.834,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking/blob/dc6a99b6b74202880e31491a445a94884c97890c/README.md#benchmarks",
    "by": "lab",
    "date": "2026-04-01",
    "note": "Arcee AI Trinity-Large-Thinking model card (pinned), Benchmarks table, row 'MMLU-Pro', column 'Trinity-Large-Thinking' printed 83.4. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "trinity-large-preview",
    "bench": "gpqa-diamond",
    "score": 0.633,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Preview/blob/d8e126719c8bf2b3ac5a3551bd1f07fcee767a90/README.md#benchmarks",
    "by": "lab",
    "date": "2026-01-27",
    "note": "Arcee AI Trinity-Large Preview model card (pinned), Benchmarks table, row 'GPQA-Diamond', column 'Trinity-Large Preview' printed 63.3. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "trinity-large-preview",
    "bench": "aime-2025",
    "score": 0.24,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Preview/blob/d8e126719c8bf2b3ac5a3551bd1f07fcee767a90/README.md#benchmarks",
    "by": "lab",
    "date": "2026-01-27",
    "note": "Arcee AI Trinity-Large Preview model card (pinned), Benchmarks table, row 'AIME 2025', column 'Trinity-Large Preview' printed 24.0. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "trinity-large-preview",
    "bench": "mmlu-pro",
    "score": 0.752,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Preview/blob/d8e126719c8bf2b3ac5a3551bd1f07fcee767a90/README.md#benchmarks",
    "by": "lab",
    "date": "2026-01-27",
    "note": "Arcee AI Trinity-Large Preview model card (pinned), Benchmarks table, row 'MMLU-Pro', column 'Trinity-Large Preview' printed 75.2. Settings not printed beyond the card ('All models evaluated in mini-swe-agent-v2' applies only to SWE-bench). Only benchmarks reported on both Arcee Trinity Large cards are recorded (others would be single-model runs). Vendor-reported (rule b): Arcee's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. Date is the model repository creation date.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "mbpp",
    "score": 0.7,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 8B Instruct', column 'MBPP pass@1' printed 70.0. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 8B Instruct', column 'HumanEval pass@1' printed 76.8. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "math-subset-unspecified",
    "score": 0.545,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 8B Instruct', column 'Math maj@1' printed 54.5. MATH subset not printed; maj@1. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "arena-hard-unspecified",
    "score": 0.709,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Chat/Arena (gpt-4o judge), row 'Ministral 8B Instruct', column 'Arena Hard' printed 70.9. Arena Hard version not printed (judge gpt-4o per the table heading). Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "mbpp",
    "score": 0.677,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 3B Instruct', column 'MBPP pass@1' printed 67.7. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "humaneval",
    "score": 0.774,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 3B Instruct', column 'HumanEval pass@1' printed 77.4. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "math-subset-unspecified",
    "score": 0.517,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Ministral 3B Instruct', column 'Math maj@1' printed 51.7. MATH subset not printed; maj@1. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "arena-hard-unspecified",
    "score": 0.643,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Chat/Arena (gpt-4o judge), row 'Ministral 3B Instruct', column 'Arena Hard' printed 64.3. Arena Hard version not printed (judge gpt-4o per the table heading). Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "mbpp",
    "score": 0.502,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Mistral 7B Instruct v0.3', column 'MBPP pass@1' printed 50.2. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "humaneval",
    "score": 0.384,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Mistral 7B Instruct v0.3', column 'HumanEval pass@1' printed 38.4. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "math-subset-unspecified",
    "score": 0.132,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Code & Math, row 'Mistral 7B Instruct v0.3', column 'Math maj@1' printed 13.2. MATH subset not printed; maj@1. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "arena-hard-unspecified",
    "score": 0.443,
    "source": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/blob/2f494a194c5b980dfb9772cb92d26cbb671fce5a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-10-15",
    "note": "Mistral AI Ministral-8B-Instruct-2410 model card (pinned 2f494a1, Mistral Research License), Benchmarks / Instruct Models / Chat/Arena (gpt-4o judge), row 'Mistral 7B Instruct v0.3', column 'Arena Hard' printed 44.3. Arena Hard version not printed (judge gpt-4o per the table heading). Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mathstral-7b-v0-1",
    "bench": "math-subset-unspecified",
    "score": 0.566,
    "source": "https://huggingface.co/mistralai/Mathstral-7B-v0.1/blob/ec3a48484ef241dfe03282edcb0f25e564923823/README.md#evaluation",
    "by": "lab",
    "date": "2024-07-16",
    "note": "Mistral AI Mathstral-7B-v0.1 model card (pinned ec3a484, Apache-2.0), Evaluation table, row 'Mathstral 7B', column 'MATH' printed 56.6 (subset and shots not printed). The open card alone would form a single-model run, so the figure is recorded under Mistral's vendor-score entry with its other own models; comparison rows (DeepSeek Math 7B, Llama3 8B, GLM4 9B, QWen2 7B, Gemma2 9B) not used because their base/instruct identity is not printed. Date is the model repository creation date. Vendor-reported (rule b): Mistral AI's own figure for its own model, hand-recorded one figure at a time; Gemma and Llama rows not used.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "aime-2025",
    "score": 0.785,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-16",
    "note": "NVIDIA NVIDIA-Nemotron-3-Nano-4B-BF16 card, Reasoning-On table, row 'AIME25' printed 78.5; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "math-500",
    "score": 0.954,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-16",
    "note": "NVIDIA NVIDIA-Nemotron-3-Nano-4B-BF16 card, Reasoning-On table, row 'MATH500' printed 95.4; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "gpqa-unspecified",
    "score": 0.532,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-16",
    "note": "NVIDIA NVIDIA-Nemotron-3-Nano-4B-BF16 card, Reasoning-On table, row 'GPQA' printed 53.2; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "livecodebench-unversioned",
    "score": 0.518,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-16",
    "note": "NVIDIA NVIDIA-Nemotron-3-Nano-4B-BF16 card, Reasoning-On table, row 'LCB' printed 51.8; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "bfcl-v3",
    "score": 0.611,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-16",
    "note": "NVIDIA NVIDIA-Nemotron-3-Nano-4B-BF16 card, Reasoning-On table, row 'BFCL v3' printed 61.1; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2025",
    "score": 0.721,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md#evaluation-results",
    "by": "lab",
    "date": "2025-08-18",
    "note": "NVIDIA NVIDIA-Nemotron-Nano-9B-v2 card, Benchmark Results (Reasoning On) table, NVIDIA-Nemotron-Nano-9B-v2 column, row 'AIME25' printed 72.1%; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "math-500",
    "score": 0.978,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md#evaluation-results",
    "by": "lab",
    "date": "2025-08-18",
    "note": "NVIDIA NVIDIA-Nemotron-Nano-9B-v2 card, Benchmark Results (Reasoning On) table, NVIDIA-Nemotron-Nano-9B-v2 column, row 'MATH500' printed 97.8%; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "gpqa-unspecified",
    "score": 0.64,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md#evaluation-results",
    "by": "lab",
    "date": "2025-08-18",
    "note": "NVIDIA NVIDIA-Nemotron-Nano-9B-v2 card, Benchmark Results (Reasoning On) table, NVIDIA-Nemotron-Nano-9B-v2 column, row 'GPQA' printed 64.0%; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-unversioned",
    "score": 0.711,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md#evaluation-results",
    "by": "lab",
    "date": "2025-08-18",
    "note": "NVIDIA NVIDIA-Nemotron-Nano-9B-v2 card, Benchmark Results (Reasoning On) table, NVIDIA-Nemotron-Nano-9B-v2 column, row 'LCB' printed 71.1%; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "bfcl-v3",
    "score": 0.669,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md#evaluation-results",
    "by": "lab",
    "date": "2025-08-18",
    "note": "NVIDIA NVIDIA-Nemotron-Nano-9B-v2 card, Benchmark Results (Reasoning On) table, NVIDIA-Nemotron-Nano-9B-v2 column, row 'BFCL v3' printed 66.9%; evaluated with NeMo-Skills in Reasoning-On mode per the card. GPQA subset and LCB window not printed. Recorded under NVIDIA's vendor-score entry so the two Nano models' identical-harness results form one NVIDIA run; the openly licensed card copies remain the single-model runs already in the data. Date is the card's stated release date. Vendor-reported (rule b): NVIDIA's own figure for its own model, hand-recorded one figure at a time; the Qwen3-8B comparison column is not used.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "mmlu",
    "score": 0.379,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'MMLU', column 'Qwen2-0.5B-Instruct' = 37.9 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "mmlu",
    "score": 0.524,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'MMLU', column 'Qwen2-1.5B-Instruct' = 52.4 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "humaneval",
    "score": 0.171,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'HumanEval', column 'Qwen2-0.5B-Instruct' = 17.1 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "humaneval",
    "score": 0.378,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'HumanEval', column 'Qwen2-1.5B-Instruct' = 37.8 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "gsm8k",
    "score": 0.401,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'GSM8K', column 'Qwen2-0.5B-Instruct' = 40.1 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "gsm8k",
    "score": 0.616,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'GSM8K', column 'Qwen2-1.5B-Instruct' = 61.6 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "c-eval",
    "score": 0.452,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'C-Eval', column 'Qwen2-0.5B-Instruct' = 45.2 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "c-eval",
    "score": 0.638,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'C-Eval', column 'Qwen2-1.5B-Instruct' = 63.8 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "ifeval-prompt-strict",
    "score": 0.2,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'IFEval (Prompt Strict-Acc.)', column 'Qwen2-0.5B-Instruct' = 20.0 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "ifeval-prompt-strict",
    "score": 0.29,
    "source": "https://huggingface.co/Qwen/Qwen2-0.5B-Instruct/blob/c540970f9e29518b1d8f06ab8b24cba66ad77b6d/README.md#evaluation",
    "by": "lab",
    "date": "2024-06-03",
    "note": "Qwen's own evaluation in the pinned Apache-2.0 Qwen2-0.5B-Instruct model card, Evaluation table, row 'IFEval (Prompt Strict-Acc.)', column 'Qwen2-1.5B-Instruct' = 29.0 (both are Qwen2 instruct models; the Qwen1.5 chat columns are not in the catalogue). Shots and prompts not printed. Date is the model repository creation date.",
    "source_id": "card2-qwen-qwen2-0-5b-instruct"
  },
  {
    "model": "openchat-3-5",
    "bench": "mmlu",
    "score": 0.643,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "lab",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenChat-3.5', column 'MMLU' = 64.3; zero-shot, chat mode (AGIEval/Orca setting). Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openchat-3-5",
    "bench": "humaneval",
    "score": 0.555,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "lab",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenChat-3.5', column 'HumanEval' = 55.5; evaluated with EvalPlus. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openchat-3-5",
    "bench": "bbh",
    "score": 0.635,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "lab",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenChat-3.5', column 'BBH CoT' = 63.5; 3-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openchat-3-5",
    "bench": "gsm8k",
    "score": 0.773,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "lab",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenChat-3.5', column 'GSM8K' = 77.3; 8-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "mmlu",
    "score": 0.638,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenHermes 2.5', column 'MMLU' = 63.8; zero-shot, chat mode (AGIEval/Orca setting). Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "humaneval",
    "score": 0.482,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenHermes 2.5', column 'HumanEval' = 48.2; evaluated with EvalPlus. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "bbh",
    "score": 0.599,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenHermes 2.5', column 'BBH CoT' = 59.9; 3-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "gsm8k",
    "score": 0.735,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'OpenHermes 2.5', column 'GSM8K' = 73.5; 8-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "mmlu",
    "score": 0.398,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'Zephyr-β', column 'MMLU' = 39.8; zero-shot, chat mode (AGIEval/Orca setting). Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. The card notes Zephyr-β 'often fails to follow few-shot CoT instructions' (low BBH CoT/GSM8K). MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "humaneval",
    "score": 0.22,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'Zephyr-β', column 'HumanEval' = 22.0; evaluated with EvalPlus. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. The card notes Zephyr-β 'often fails to follow few-shot CoT instructions' (low BBH CoT/GSM8K). MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "bbh",
    "score": 0.16,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'Zephyr-β', column 'BBH CoT' = 16.0; 3-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. The card notes Zephyr-β 'often fails to follow few-shot CoT instructions' (low BBH CoT/GSM8K). MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5",
    "invalid": "Few-shot CoT format failure in a chat-mode harness: 16.0 vs 44.9 for the same checkpoint in the Tulu 2 paper (3-shot CoT), and below BBH's published random average of 25.7."
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "gsm8k",
    "score": 0.051,
    "source": "https://huggingface.co/openchat/openchat_3.5/blob/0fc98e324280bc4bf5d2c30ecf7b97b84fb8a19b/README.md#benchmarks",
    "by": "independent",
    "date": "2023-10-30",
    "note": "OpenChat 3.5 model card (pinned 0fc98e3, Apache-2.0), Benchmarks table, row 'Zephyr-β', column 'GSM8K' = 5.1; 8-shot CoT with the Chain-of-Thought Hub configuration. Card: 'All models are evaluated in chat mode'; only the ChatGPT, Mistral and Open-source SOTA rows are marked as taken from reports, so the OpenChat-3.5, OpenHermes 2.5 and Zephyr-β rows are the OpenChat team's runs. The card notes Zephyr-β 'often fails to follow few-shot CoT instructions' (low BBH CoT/GSM8K). MT-Bench, AGIEval, BBH MC and TruthfulQA columns not used (no exact id or subset not printed). Date is the model repository creation date.",
    "source_id": "card2-openchat-openchat-3-5",
    "invalid": "Same few-shot CoT format failure: 5.1 vs 28.0 in the Tulu 2 paper's 8-shot CoT run."
  },
  {
    "model": "llama-2-7b",
    "bench": "mmlu",
    "score": 0.4432,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'MMLU' = 44.32; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "c-eval",
    "score": 0.3242,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'C-Eval' = 32.42; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "humaneval",
    "score": 0.122,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'HumanEval' = 12.20; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "mbpp",
    "score": 0.2717,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'MBPP' = 27.17; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "gsm8k",
    "score": 0.1357,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'GSM8K' = 13.57; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "bbh",
    "score": 0.3323,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'BBH' = 33.23; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-7b",
    "bench": "arc-challenge",
    "score": 0.4275,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-7B', column 'ARC-c' = 42.75; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "mmlu",
    "score": 0.5471,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'MMLU' = 54.71; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "c-eval",
    "score": 0.3732,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'C-Eval' = 37.32; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "humaneval",
    "score": 0.1707,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'HumanEval' = 17.07; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "mbpp",
    "score": 0.3255,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'MBPP' = 32.55; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "gsm8k",
    "score": 0.2115,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'GSM8K' = 21.15; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "bbh",
    "score": 0.3792,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'BBH' = 37.92; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b",
    "bench": "arc-challenge",
    "score": 0.5819,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Llama2-13B', column 'ARC-c' = 58.19; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mmlu",
    "score": 0.6269,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'MMLU' = 62.69; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "c-eval",
    "score": 0.4612,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'C-Eval' = 46.12; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "humaneval",
    "score": 0.2744,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'HumanEval' = 27.44; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mbpp",
    "score": 0.452,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'MBPP' = 45.20; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "gsm8k",
    "score": 0.3313,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'GSM8K' = 33.13; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "bbh",
    "score": 0.4106,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'BBH' = 41.06; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-challenge",
    "score": 0.7073,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Mistral-7B', column 'ARC-c' = 70.73; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "mmlu",
    "score": 0.5353,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'MMLU' = 53.53; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "c-eval",
    "score": 0.4029,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'C-Eval' = 40.29; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "humaneval",
    "score": 0.2439,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'HumanEval' = 24.39; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "mbpp",
    "score": 0.3653,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'MBPP' = 36.53; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "gsm8k",
    "score": 0.2244,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'GSM8K' = 22.44; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "bbh",
    "score": 0.3624,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'BBH' = 36.24; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "falcon-40b",
    "bench": "arc-challenge",
    "score": 0.5768,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Falcon-40B', column 'ARC-c' = 57.68; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "mmlu",
    "score": 0.5266,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'MMLU' = 52.66; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "c-eval",
    "score": 0.2337,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'C-Eval' = 23.37; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "humaneval",
    "score": 0.4756,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'HumanEval' = 47.56; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "mbpp",
    "score": 0.5504,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'MBPP' = 55.04; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "gsm8k",
    "score": 0.5716,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'GSM8K' = 57.16; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "bbh",
    "score": 0.4339,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'BBH' = 43.39; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "phi-2",
    "bench": "arc-challenge",
    "score": 0.7125,
    "source": "https://arxiv.org/html/2404.06395v3",
    "by": "independent",
    "date": "2024-06-03",
    "note": "MiniCPM paper (Hu et al., arXiv:2404.06395v3, CC BY 4.0), Table 3 'Benchmark Score of MiniCPM-2.4B and MiniCPM-1.2B', row 'Phi-2(2B)', column 'ARC-c' = 71.25; all rows except Gemini Nano-3B (borrowed) were run by the authors with their open-source UltraEval tool (section 6.5) on vLLM, adapting prompts per model. Pretrained (base) checkpoints. Gemma rows not used (base vs instruct not printed).",
    "source_id": "paper-2404.06395v3"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "agieval-en",
    "score": 0.3885,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column AGI (zero-shot average of the nine English AGIEval tasks in Table 6), row 'LLaMA-2-Chat-13B' = 38.85. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "bbh",
    "score": 0.336,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column BBH (zero-shot), row 'LLaMA-2-Chat-13B' = 33.6. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "gsm8k",
    "score": 0.2509,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column GSM8K (zero-shot), row 'LLaMA-2-Chat-13B' = 25.09. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "mmlu",
    "score": 0.4914,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column MMLU (zero-shot), row 'LLAMA-2-Chat-13B' = 49.14. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "arc-easy",
    "score": 0.7626,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Easy (zero-shot), row 'LLAMA-2-Chat-13B' = 76.26. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "arc-challenge",
    "score": 0.6118,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Challenge (zero-shot), row 'LLAMA-2-Chat-13B' = 61.18. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "agieval-en",
    "score": 0.3825,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column AGI (zero-shot average of the nine English AGIEval tasks in Table 6), row 'WizardLM-13B' = 38.25. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "bbh",
    "score": 0.3847,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column BBH (zero-shot), row 'WizardLM-13B' = 38.47. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "gsm8k",
    "score": 0.486,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column GSM8K (zero-shot), row 'WizardLM-13B' = 48.60. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "mmlu",
    "score": 0.4281,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column MMLU (zero-shot), row 'WizardLM-13B' = 42.81. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "arc-easy",
    "score": 0.6898,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Easy (zero-shot), row 'WizardLM-13B' = 68.98. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "arc-challenge",
    "score": 0.5043,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Challenge (zero-shot), row 'WizardLM-13B' = 50.43. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "agieval-en",
    "score": 0.467,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column AGI (zero-shot average of the nine English AGIEval tasks in Table 6), row 'LLaMA-2-Chat-70B' = 46.70. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "bbh",
    "score": 0.4468,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column BBH (zero-shot), row 'LLaMA-2-Chat-70B' = 44.68. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "gsm8k",
    "score": 0.5201,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column GSM8K (zero-shot), row 'LLaMA-2-Chat-70B' = 52.01. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "mmlu",
    "score": 0.5854,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column MMLU (zero-shot), row 'LLAMA-2-Chat-70B' = 58.54. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "arc-easy",
    "score": 0.822,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Easy (zero-shot), row 'LLAMA-2-Chat-70B' = 82.20. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "arc-challenge",
    "score": 0.6766,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Challenge (zero-shot), row 'LLAMA-2-Chat-70B' = 67.66. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "agieval-en",
    "score": 0.4873,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column AGI (zero-shot average of the nine English AGIEval tasks in Table 6), row 'WizardLM-70B' = 48.73. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "bbh",
    "score": 0.5108,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column BBH (zero-shot), row 'WizardLM-70B' = 51.08. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "gsm8k",
    "score": 0.7324,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 1 column GSM8K (zero-shot), row 'WizardLM-70B' = 73.24. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "mmlu",
    "score": 0.55,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column MMLU (zero-shot), row 'WizardLM-70B' = 55.00. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "arc-easy",
    "score": 0.8068,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Easy (zero-shot), row 'WizardLM-70B' = 80.68. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "arc-challenge",
    "score": 0.7193,
    "source": "https://arxiv.org/html/2311.11045v2",
    "by": "independent",
    "date": "2023-11-21",
    "note": "Orca 2 paper (Mitra et al., Microsoft, arXiv:2311.11045v2, CC BY 4.0), Table 2 column ARC Challenge (zero-shot), row 'WizardLM-70B' = 71.93. Baselines run by the authors; section 4 identifies the checkpoints: Llama-2-13b-chat-hf / Llama-2-70b-chat-hf and WizardLM-13B-V1.2 / WizardLM-70B-V1.0. Empty system message rows. Orca 2 models are not in the catalogue; ChatGPT/GPT-4 rows not used (unversioned).",
    "source_id": "paper-2311.11045v2"
  },
  {
    "model": "flux-3-video",
    "bench": "lmarena-video-i2v",
    "score": 1449.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/image_to_video/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'flux-3-video-20260811' (organization bfl); 37,042 votes; published 95% interval 1444.5–1454.4; leaderboard_publish_date 2026-09-22. Dated arena label for BFL's FLUX 3 Video API model (one model on POST /v1/flux-3-video with t2v/i2v modes, generally available 4 Aug 2026); the text_to_video entry 'flux-3-video' is already mapped to this model. File identical at main (c60d8c87, same sha256 55fd7438...).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "minimax-h3",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1324.98,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "minimax-h3; Overall ranking ELO 1324.98; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1315.7,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "grok-imagine-1.5; Overall ranking ELO 1315.7; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "kling-3-0-pro",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1290.87,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "kling-3.0-pro; Overall ranking ELO 1290.87; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1288.05,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "gemini-omni-flash-1.1; Overall ranking ELO 1288.05; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "flux-3-video",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1277.01,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "flux-3-video; Overall ranking ELO 1277.01; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "veo-3-1",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1266.6,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "veo-3.1; Overall ranking ELO 1266.6; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "wan-3-0",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1249.97,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "wan-3.0; Overall ranking ELO 1249.97; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "seedance-2-5",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1175.61,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "seedance-2.5; Overall ranking ELO 1175.61; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "happyhorse-1-1",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1137.92,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "happy-horse-1.1; Overall ranking ELO 1137.92; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "pixverse-v5-6",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1085.73,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "pixverse-v5.6; Overall ranking ELO 1085.73; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "vidu-q3-pro",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1083.15,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "vidu-q3-pro; Overall ranking ELO 1083.15; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "cosmos3-nano",
    "bench": "rapidata-i2v-realism-overall",
    "score": 717.128,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "cosmos-3-nano; Overall ranking ELO 717.128; Rapidata I2V realism & visual quality, 25 prompts, 522,210 votes; generation settings not stated on card",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1390.7,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Gemini Omni Flash 1.1; Alignment-leaderboard ELO 1390.7; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "wan-3-0",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1383.4,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Wan 3.0; Alignment-leaderboard ELO 1383.4; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "happyhorse-1-1",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1290.8,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Happy Horse 1.1; Alignment-leaderboard ELO 1290.8; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1226,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Grok Imagine Video 1.5; Alignment-leaderboard ELO 1226.0; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "minimax-h3",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1152.9,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "MiniMax H3; Alignment-leaderboard ELO 1152.9; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "veo-3-1",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1114.3,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Veo 3.1; Alignment-leaderboard ELO 1114.3; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "seedance-2-5",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1076.3,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Dreamina Seedance 2.5; Alignment-leaderboard ELO 1076.3; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "flux-3-video",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1058.2,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "FLUX 3 Video; Alignment-leaderboard ELO 1058.2; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "kling-3-0-pro",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1000,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Kling 3.0 Pro; Alignment-leaderboard ELO 1000.0; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "vidu-q3-pro",
    "bench": "rapidata-relative-camera-i2v",
    "score": 977.9,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Vidu Q3 Pro; Alignment-leaderboard ELO 977.9; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "cosmos3-nano",
    "bench": "rapidata-relative-camera-i2v",
    "score": 717.5,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Cosmos 3 Nano; Alignment-leaderboard ELO 717.5; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "pixverse-v5-6",
    "bench": "rapidata-relative-camera-i2v",
    "score": 703.5,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Pixverse V5.6; Alignment-leaderboard ELO 703.5; 220 scene-specific camera prompts on 50 stills, I2V, 900,318 votes",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "flux-3-video",
    "bench": "rapidata-camera-i2v",
    "score": 619.2,
    "source": "https://huggingface.co/datasets/Rapidata/camera-movement-human-preference-324k/blob/6206acfab997d39eaf63012d127c917de7995cc9/README.md#overall-ranking-elo-on-the-movement-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "Flux 3; Movement ELO 619.2; commercial I2V via public API at 720p (other commercial models 1080p), clips for 598 of 600 prompts; added in the 6206acf revision",
    "source_id": "visual-rapid-camera"
  },
  {
    "model": "sd-1-4",
    "bench": "heim-alignment-winrate",
    "score": 0.691,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "Stable Diffusion v1.4 (1B); Table 5 Alignment column 0.691 (win rate among 26 HEIM models); Table 4 'Stable Diffusion v1-4' (CompVis), ref [4]; Appendix D",
    "source_id": "visual-heim"
  },
  {
    "model": "sd-1-5",
    "bench": "heim-alignment-winrate",
    "score": 0.531,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "Stable Diffusion v1.5 (1B); Table 5 Alignment column 0.531 (win rate among 26 HEIM models); Table 4 'Stable Diffusion v1-5' (creator Runway), ref [4]",
    "source_id": "visual-heim"
  },
  {
    "model": "dreamlike-photoreal-2-0",
    "bench": "heim-alignment-winrate",
    "score": 0.851,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "Dreamlike Photoreal v2.0 (1B); Table 5 Alignment column 0.851 (win rate among 26 HEIM models); Table 4 'Dreamlike Photoreal 2.0', ref [46] = huggingface.co/dreamlike-art/dreamlike-photoreal-2.0",
    "source_id": "visual-heim"
  },
  {
    "model": "openjourney",
    "bench": "heim-alignment-winrate",
    "score": 0.617,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "Openjourney v1 (1B); Table 5 Alignment column 0.617 (win rate among 26 HEIM models); Table 4 'Openjourney', ref [47] = huggingface.co/prompthero/openjourney; Appendix D 'Openjourney {v1, v4}' (v4 = ref [48], a different repo, not mapped)",
    "source_id": "visual-heim"
  },
  {
    "model": "dall-e-2",
    "bench": "heim-alignment-winrate",
    "score": 0.971,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "DALL-E 2 (3.5B); Table 5 Alignment column 0.971 (win rate among 26 HEIM models); Table 4 'DALL-E 2' (OpenAI, limited access), ref [3]",
    "source_id": "visual-heim"
  },
  {
    "model": "dall-e-mega-min-dalle",
    "bench": "heim-alignment-winrate",
    "score": 0.589,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "DALL-E mega (2.6B); Table 5 Alignment column 0.589 (win rate among 26 HEIM models); Table 4 'DALL-E mega' (craiyon, 2.6B), ref [52] = github.com/borisdayma/dalle-mini; Appendix D: mini and mega variants have 0.4B and 2.6B parameters. Same DALL-E Mega weights the catalog entry denotes (kuprel/min-dalle is a port of these weights); HEIM's separate 'minDALL-E' row (Kakao Brain 1.3B, ref [53]) is a different model and is NOT mapped",
    "source_id": "visual-heim"
  },
  {
    "model": "cogview2",
    "bench": "heim-alignment-winrate",
    "score": 0.074,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "CogView2 (6B); Table 5 Alignment column 0.074 (win rate among 26 HEIM models); Table 4 'CogView2' (Tsinghua), ref [10]",
    "source_id": "visual-heim"
  },
  {
    "model": "deepfloyd-if-m",
    "bench": "heim-alignment-winrate",
    "score": 0.566,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "DeepFloyd IF Medium (0.4B); Table 5 Alignment column 0.566 (win rate among 26 HEIM models); Table 4 'DeepFloyd-IF M v1.0', ref [55]",
    "source_id": "visual-heim"
  },
  {
    "model": "deepfloyd-if-l",
    "bench": "heim-alignment-winrate",
    "score": 0.514,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "DeepFloyd IF Large (0.9B); Table 5 Alignment column 0.514 (win rate among 26 HEIM models); Table 4 'DeepFloyd-IF L v1.0', ref [55]",
    "source_id": "visual-heim"
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "heim-alignment-winrate",
    "score": 0.589,
    "source": "https://arxiv.org/html/2311.04287v1#S7.T5",
    "by": "independent",
    "date": "2023-11-07",
    "note": "DeepFloyd IF X-Large (4.3B); Table 5 Alignment column 0.589 (win rate among 26 HEIM models); Table 4 'DeepFloyd-IF XL v1.0', ref [55]",
    "source_id": "visual-heim"
  },
  {
    "model": "dall-e-2",
    "bench": "imagereward-human-wins",
    "score": 390,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "independent",
    "date": "2023-12-28",
    "note": "DALL-E 2; Table 1 Real User Prompts Human Eval. #Win 390 (rank 3 of 6); Sec. 2.3: 'DALL-E 2 (via OpenAI API) [42]'",
    "source_id": "visual-imagereward"
  },
  {
    "model": "sd-1-4",
    "bench": "imagereward-human-wins",
    "score": 362,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Stable Diffusion 1.4; Table 1 Real User Prompts Human Eval. #Win 362 (rank 4 of 6); Sec. 2.3: 'Stable Diffusion (SD) 1.4 ... [45]'",
    "source_id": "visual-imagereward"
  },
  {
    "model": "sd-1-4",
    "bench": "imagereward-coco30k-fid",
    "score": 17.9,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Stable Diffusion 1.4; Table 1 MS-COCO 2014 Zero-shot FID 17.9 (author-computed, not marked *); 30k captions, 256x256, CFG grid search; Sec. 2.3: 'Stable Diffusion (SD) 1.4 ... [45]'",
    "source_id": "visual-imagereward"
  },
  {
    "model": "versatile-diffusion",
    "bench": "imagereward-human-wins",
    "score": 340,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Versatile Diffusion; Table 1 Real User Prompts Human Eval. #Win 340 (rank 5 of 6); Sec. 2.3: 'Versatile Diffusion (VD) [62]'; single public release shi-labs/versatile-diffusion v1.0",
    "source_id": "visual-imagereward"
  },
  {
    "model": "versatile-diffusion",
    "bench": "imagereward-coco30k-fid",
    "score": 18.4,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Versatile Diffusion; Table 1 MS-COCO 2014 Zero-shot FID 18.4 (author-computed, not marked *); 30k captions, 256x256, CFG grid search; Sec. 2.3: 'Versatile Diffusion (VD) [62]'; single public release shi-labs/versatile-diffusion v1.0",
    "source_id": "visual-imagereward"
  },
  {
    "model": "cogview2",
    "bench": "imagereward-human-wins",
    "score": 74,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "lab",
    "date": "2023-12-28",
    "note": "CogView 2; Table 1 Real User Prompts Human Eval. #Win 74 (rank 6 of 6); Sec. 2.3: 'CogView 2 [12]' (Ding et al., NeurIPS 2022); single public release THUDM/CogView2",
    "source_id": "visual-imagereward"
  },
  {
    "model": "cogview2",
    "bench": "imagereward-coco30k-fid",
    "score": 26.2,
    "source": "https://arxiv.org/html/2304.05977v4#S2.T1",
    "by": "lab",
    "date": "2023-12-28",
    "note": "CogView 2; Table 1 MS-COCO 2014 Zero-shot FID 26.2 (author-computed, not marked *); 30k captions, 256x256, CFG grid search; Sec. 2.3: 'CogView 2 [12]' (Ding et al., NeurIPS 2022); single public release THUDM/CogView2",
    "source_id": "visual-imagereward"
  },
  {
    "model": "gpt-image-2",
    "bench": "vvrbench-accuracy",
    "score": 0.8686,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'GPT-Image-2'; Table 3 VVRBench Accuracy (%) 86.86±0.68 -> 86.86/100; GPT-Image-2 (2026-04-21) via API, medium quality 1024x1024 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "gpt-image-1-mini",
    "bench": "vvrbench-accuracy",
    "score": 0.264,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'GPT-Image-1-mini'; Table 3 VVRBench Accuracy (%) 26.40±0.87 -> 26.40/100; API, medium quality 1024x1024 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "flux-2-dev",
    "bench": "vvrbench-accuracy",
    "score": 0.1915,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'FLUX.2-dev'; Table 3 VVRBench Accuracy (%) 19.15±0.78 -> 19.15/100; open weights, 50 steps, guidance 4.0 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "hunyuanimage-2-1",
    "bench": "vvrbench-accuracy",
    "score": 0.1879,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'HunyuanImage-2.1'; Table 3 VVRBench Accuracy (%) 18.79±0.78 -> 18.79/100; open weights, 50 steps, guidance 3.5 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "qwen-image-2512",
    "bench": "vvrbench-accuracy",
    "score": 0.0579,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Qwen-Image-2512'; Table 3 VVRBench Accuracy (%) 5.79±0.47 -> 5.79/100; open weights, 50 steps, guidance 4.0 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "hidream-i1-full",
    "bench": "vvrbench-accuracy",
    "score": 0.0422,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'HiDream-I1-Full'; Table 3 VVRBench Accuracy (%) 4.22±0.41 -> 4.22/100; open weights, 50 steps, guidance 5.0 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "flux-1-dev",
    "bench": "vvrbench-accuracy",
    "score": 0.0387,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'FLUX.1-dev'; Table 3 VVRBench Accuracy (%) 3.87±0.40 -> 3.87/100; open weights, 28 steps, guidance 3.5 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "flux-1-schnell",
    "bench": "vvrbench-accuracy",
    "score": 0.0288,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'FLUX.1-schnell'; Table 3 VVRBench Accuracy (%) 2.88±0.35 -> 2.88/100; open weights, 4 steps, guidance 0.0 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "sd-3-5-medium",
    "bench": "vvrbench-accuracy",
    "score": 0.0281,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'SD3.5 Medium'; Table 3 VVRBench Accuracy (%) 2.81±0.34 -> 2.81/100; open weights, 40 steps, guidance 4.5 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "sd-3-5-large",
    "bench": "vvrbench-accuracy",
    "score": 0.0245,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'SD3.5 Large'; Table 3 VVRBench Accuracy (%) 2.45±0.32 -> 2.45/100; open weights, 40 steps, guidance 4.5 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "sdxl",
    "bench": "vvrbench-accuracy",
    "score": 0.0002,
    "source": "https://arxiv.org/html/2609.35641v1#S3.T3",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'SDXL 1.0'; Table 3 VVRBench Accuracy (%) 0.02±0.05 -> 0.02/100; open weights, 40 steps, guidance 5.0 (Table 9); 10,000 tasks, all-constraints-pass verifier accuracy.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "gpt-image-2-5-sunburst",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.2139,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'GPT-Image-2.5-Sunburst'; Table 4 VVRBench-Challenge Accuracy 21.39±3.14 -> 21.39/100; GPT-Image-2.5-Sunburst (2026-09-08) via API, medium quality 1024x1024 (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "gpt-image-2",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.1028,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'GPT-Image-2'; Table 4 VVRBench-Challenge Accuracy 10.28±2.43 -> 10.28/100; GPT-Image-2 (2026-04-21) via API, medium quality 1024x1024 (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "nano-banana-2-lite",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.0736,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Gemini-3.1-Flash-Lite-Image'; Table 4 VVRBench-Challenge Accuracy 7.36±2.14 -> 7.36/100; Gemini API, 1K (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "nano-banana-pro",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.0458,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Gemini-3-Pro-Image'; Table 4 VVRBench-Challenge Accuracy 4.58±1.78 -> 4.58/100; Gemini API, 1K (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "nano-banana-2",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.0389,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Gemini-3.1-Flash-Image'; Table 4 VVRBench-Challenge Accuracy 3.89±1.67 -> 3.89/100; Gemini API, 1K (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "nano-banana",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.0111,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Gemini-2.5-Flash-Image'; Table 4 VVRBench-Challenge Accuracy 1.11±1.07 -> 1.11/100; Gemini API, model default (1K) (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "gpt-image-1-mini",
    "bench": "vvrbench-challenge-accuracy",
    "score": 0.0028,
    "source": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'GPT-Image-1-mini'; Table 4 VVRBench-Challenge Accuracy 0.28±0.73 -> 0.28/100; API, medium quality 1024x1024 (Table 9); 720 tasks, complexity 45-80.",
    "source_id": "visual-vvrbench"
  },
  {
    "model": "qwen-image-2-0-pro",
    "bench": "chartsync-overall",
    "score": 0.6554,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Qwen-Image-2.0-Pro'; Table 2 ChartSync Overall 65.54 -> 65.54/100; Proprietary API; release label 'Qwen-Image-2.0-Pro' retained, no API snapshot inferred (same mapping as existing Qwen-Image-Bench/FEPBench rows).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "chartsync-overall",
    "score": 0.7109,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'SeeDream-5.0-Lite'; Table 2 ChartSync Overall 71.09 -> 71.09/100; Proprietary API.",
    "source_id": "visual-chartsync"
  },
  {
    "model": "wan-2-7-image-pro",
    "bench": "chartsync-overall",
    "score": 0.7447,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Wan2.7-Image-Pro'; Table 2 ChartSync Overall 74.47 -> 74.47/100; Proprietary API.",
    "source_id": "visual-chartsync"
  },
  {
    "model": "gpt-image-2",
    "bench": "chartsync-overall",
    "score": 0.8178,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'GPT-Image-2'; Table 2 ChartSync Overall 81.78 -> 81.78/100; footnote model identifier gpt-image-2-2026-04-21, accessed July 2026.",
    "source_id": "visual-chartsync"
  },
  {
    "model": "nano-banana-pro",
    "bench": "chartsync-overall",
    "score": 0.8776,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Nano'; Table 2 ChartSync Overall 87.76 -> 87.76/100; footnote model identifier gemini-3-pro-image, accessed July 2026.",
    "source_id": "visual-chartsync"
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "chartsync-overall",
    "score": 0.459,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Step1X-Edit-v1p2'; Table 2 ChartSync Overall 45.90 -> 45.90/100; open weights, guidance 6.0, 50 steps (Table 7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "longcat-image-edit",
    "bench": "chartsync-overall",
    "score": 0.4725,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'LongCat-Image-Edit'; Table 2 ChartSync Overall 47.25 -> 47.25/100; open weights, guidance 4.5, 50 steps (Table 7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "qwen-image-edit",
    "bench": "chartsync-overall",
    "score": 0.5075,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Qwen-Image-Edit'; Table 2 ChartSync Overall 50.75 -> 50.75/100; open weights, guidance 4.0, 40 steps (Table 7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "chartsync-overall",
    "score": 0.6029,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'Qwen-Image-Edit-2511'; Table 2 ChartSync Overall 60.29 -> 60.29/100; open weights, guidance 4.0, 40 steps (Table 7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "gpt-image-2",
    "bench": "infoedit-avg-sr",
    "score": 0.62,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'GPT-Image-2'; Table 2 InfoEdit Avg. success rate 62.0% -> 62.0/100; API model gpt-image-2 (Table 17).",
    "source_id": "visual-infoedit"
  },
  {
    "model": "nano-banana-pro",
    "bench": "infoedit-avg-sr",
    "score": 0.416,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'Nanobanana-Pro'; Table 2 InfoEdit Avg. success rate 41.6% -> 41.6/100; API model gemini-3-pro-image-preview (Table 17).",
    "source_id": "visual-infoedit"
  },
  {
    "model": "nano-banana-2",
    "bench": "infoedit-avg-sr",
    "score": 0.38,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'Nanobanana-2'; Table 2 InfoEdit Avg. success rate 38.0% -> 38.0/100; API model gemini-3.1-flash-image-preview (Table 17).",
    "source_id": "visual-infoedit"
  },
  {
    "model": "nano-banana",
    "bench": "infoedit-avg-sr",
    "score": 0.035,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'Nanobanana'; Table 2 InfoEdit Avg. success rate 3.5% -> 3.5/100; API model gemini-2.5-flash-image (Table 17).",
    "source_id": "visual-infoedit"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "infoedit-avg-sr",
    "score": 0.05,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'Seedream-5.0-Lite'; Table 2 InfoEdit Avg. success rate 5.0% -> 5.0/100; API model doubao-seedream-5-0-260128 (Table 17).",
    "source_id": "visual-infoedit"
  },
  {
    "model": "hunyuanimage-3-0-instruct",
    "bench": "infoedit-avg-sr",
    "score": 0.032,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'HunyuanImage-3.0'; Table 2 InfoEdit Avg. success rate 3.2% -> 3.2/100; Table 17 lists the weights as huggingface.co/tencent/HunyuanImage-3.0-Instruct.",
    "source_id": "visual-infoedit"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "infoedit-avg-sr",
    "score": 0.004,
    "source": "https://arxiv.org/html/2609.33286v1#S3.T2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "'Qwen-Image-Edit'; Table 2 InfoEdit Avg. success rate 0.4% -> 0.4/100; Table 17 lists the weights as huggingface.co/Qwen/Qwen-Image-Edit-2511.",
    "source_id": "visual-infoedit"
  },
  {
    "model": "nano-banana-2",
    "bench": "paintbench-miou",
    "score": 0.171,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'NB-2'; Table 1 PaintBench Benchmark Avg. mIoU 17.1% (±0.7) -> /100; Nano-Banana-2 (ref [4] Gemini 3.1 Flash Image), provider default API sampling.",
    "source_id": "visual-paintbench"
  },
  {
    "model": "gpt-image-2",
    "bench": "paintbench-miou",
    "score": 0.163,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'GPT-I2'; Table 1 PaintBench Benchmark Avg. mIoU 16.3% (±0.7) -> /100; GPT-Image-2 (ref [3] ChatGPT Images 2.0 system card), provider default API sampling.",
    "source_id": "visual-paintbench"
  },
  {
    "model": "nano-banana",
    "bench": "paintbench-miou",
    "score": 0.111,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'NB-1'; Table 1 PaintBench Benchmark Avg. mIoU 11.1% (±0.6) -> /100; Nano-Banana-1 (ref [38] Gemini 2.5 Flash Image), provider default API sampling.",
    "source_id": "visual-paintbench"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "paintbench-miou",
    "score": 0.067,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'Qwen-IE'; Table 1 PaintBench Benchmark Avg. mIoU 6.7% (±0.5) -> /100; Qwen-Image-Edit-2511, 50 steps, true_cfg 4.0 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "bagel",
    "bench": "paintbench-miou",
    "score": 0.05,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'BAGEL'; Table 1 PaintBench Benchmark Avg. mIoU 5.0% (±0.5) -> /100; BAGEL, 50 steps, dual CFG text 4.0 / image 2.0 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "flux-2-dev",
    "bench": "paintbench-miou",
    "score": 0.046,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'FLUX.2-D'; Table 1 PaintBench Benchmark Avg. mIoU 4.6% (±0.4) -> /100; FLUX.2-dev, 50 steps, CFG 4.0 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "paintbench-miou",
    "score": 0.039,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'FLUX.1-Kt'; Table 1 PaintBench Benchmark Avg. mIoU 3.9% (±0.3) -> /100; FLUX.1-Kontext-dev, 28 steps, CFG 3.5 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "longcat-image-edit",
    "bench": "paintbench-miou",
    "score": 0.036,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'LCat-IE'; Table 1 PaintBench Benchmark Avg. mIoU 3.6% (±0.4) -> /100; LongCat-Image-Edit, 50 steps, true_cfg 4.0 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "paintbench-miou",
    "score": 0.035,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'FLUX.2-Kl'; Table 1 PaintBench Benchmark Avg. mIoU 3.5% (±0.3) -> /100; FLUX.2-klein-9B (distilled; Table 8), 50 steps, CFG 4.0 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "instructpix2pix",
    "bench": "paintbench-miou",
    "score": 0.003,
    "source": "https://arxiv.org/html/2606.00188v1#S4.T1",
    "by": "independent",
    "date": "2026-05-29",
    "note": "'IP2P'; Table 1 PaintBench Benchmark Avg. mIoU 0.3% (±0.1) -> /100; InstructPix2Pix, 100 steps, CFG 7.5 / image 1.5 (Table 9).",
    "source_id": "visual-paintbench"
  },
  {
    "model": "wan-2-7",
    "bench": "mechverse-wmb-sum",
    "score": 0.759,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'Wan2.7'; Table 1 MechVerse WorldModelBench-judge Sum 7.59 (of 10) -> 7.59/10; proprietary I2V, 'Wan 2.7 [37]'.",
    "source_id": "visual-mechverse"
  },
  {
    "model": "happyhorse-1-0",
    "bench": "mechverse-wmb-sum",
    "score": 0.756,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'Happy Horse'; Table 1 MechVerse WorldModelBench-judge Sum 7.56 (of 10) -> 7.56/10; proprietary I2V, text names 'Happy Horse 1.0 [12]'.",
    "source_id": "visual-mechverse"
  },
  {
    "model": "kling-3-0",
    "bench": "mechverse-wmb-sum",
    "score": 0.766,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'Kling3'; Table 1 MechVerse WorldModelBench-judge Sum 7.66 (of 10) -> 7.66/10; proprietary I2V, 'Kling 3 [17]'; tier not specified, so not mapped to a Pro/Standard tier (same practice as the VGI-Bench row).",
    "source_id": "visual-mechverse"
  },
  {
    "model": "cogvideox-5b-i2v",
    "bench": "mechverse-wmb-sum",
    "score": 0.579,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'CogVideoX-1.0 (5B)'; Table 1 MechVerse WorldModelBench-judge Sum 5.79 (of 10) -> 5.79/10; open-source I2V run; CogVideoX 1.0 5B in image-to-video mode, i.e. the CogVideoX-5b-I2V checkpoint (the only 1.0 5B I2V release).",
    "source_id": "visual-mechverse"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "mechverse-wmb-sum",
    "score": 0.756,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'HunyuanVideo-1.5 (8.3B)'; Table 1 MechVerse WorldModelBench-judge Sum 7.56 (of 10) -> 7.56/10; open-source I2V run of HunyuanVideo-1.5 (8.3B).",
    "source_id": "visual-mechverse"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "mechverse-wmb-sum",
    "score": 0.656,
    "source": "https://arxiv.org/html/2605.14843v1#S4.T1",
    "by": "independent",
    "date": "2026-05-14",
    "note": "'Wan2.2 (5B)'; Table 1 MechVerse WorldModelBench-judge Sum 6.56 (of 10) -> 6.56/10; open-source Wan2.2-TI2V-5B in I2V mode (not the authors' MechVerse fine-tune, which is a separate row).",
    "source_id": "visual-mechverse"
  },
  {
    "model": "gpt-image-2-5-sunburst",
    "bench": "rapidata-benchai-image-overall",
    "score": 1238.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'GPT Image 2.5 (Sunburst)' (OpenAI); Overall ranking #1, Overall ELO 1238.3 (aggregated across Alignment 1326, Preference 1250, Coherence 1246, Text coherence 1181). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "gpt-image-2-5-flare",
    "bench": "rapidata-benchai-image-overall",
    "score": 1207.1,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'GPT Image 2.5 (Flare)' (OpenAI); Overall ranking #2, Overall ELO 1207.1 (aggregated across Alignment 1341, Preference 1196, Coherence 1219, Text coherence 1136). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "gpt-image-2",
    "bench": "rapidata-benchai-image-overall",
    "score": 1167.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'GPT Image 2.0' (OpenAI); Overall ranking #3, Overall ELO 1167.3 (aggregated across Alignment 1242, Preference 1135, Coherence 1113, Text coherence 1186). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "reve-2-1",
    "bench": "rapidata-benchai-image-overall",
    "score": 1147.5,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Reve 2.1' (Reve AI); Overall ranking #4, Overall ELO 1147.5 (aggregated across Alignment 1149, Preference 1096, Coherence 1157, Text coherence 1193). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "rapidata-benchai-image-overall",
    "score": 1131.9,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'GPT Image 1.5 (high)' (OpenAI); Overall ranking #6, Overall ELO 1131.9 (aggregated across Alignment 1247, Preference 1242, Coherence 896, Text coherence 1154); quality=high setting of GPT Image 1.5. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "nano-banana-2",
    "bench": "rapidata-benchai-image-overall",
    "score": 1128.1,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'nano-banana-2' (Google); Overall ranking #7, Overall ELO 1128.1 (aggregated across Alignment 1169, Preference 1242, Coherence 1050, Text coherence 1051). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "grok-imagine-image-2",
    "bench": "rapidata-benchai-image-overall",
    "score": 1107.5,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Grok Imagine Image 2.0' (SpaceXAI); Overall ranking #8, Overall ELO 1107.5 (aggregated across Alignment 1128, Preference 1086, Coherence 1076, Text coherence 1155); the plain entry; the separate '(Medium)' and '(Thinking)' setting variants are not mapped. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "muse-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 1096.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Muse Image' (Meta); Overall ranking #9, Overall ELO 1096.3 (aggregated across Alignment 1171, Preference 1108, Coherence 1062, Text coherence 1034). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "nano-banana-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 1071.6,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'nano-banana-pro' (Google); Overall ranking #11, Overall ELO 1071.6 (aggregated across Alignment 1070, Preference 1044, Coherence 1104, Text coherence 1049). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "seedream-4-5",
    "bench": "rapidata-benchai-image-overall",
    "score": 1046.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Seedream 4.5' (ByteDance); Overall ranking #12, Overall ELO 1046.7 (aggregated across Alignment 1032, Preference 999, Coherence 948, Text coherence 1220). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "mai-image-2-5",
    "bench": "rapidata-benchai-image-overall",
    "score": 1042.1,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'MAI-Image 2.5' (Microsoft AI); Overall ranking #14, Overall ELO 1042.1 (aggregated across Alignment 1060, Preference 1114, Coherence 975, Text coherence 1015). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "imagen-4-ultra",
    "bench": "rapidata-benchai-image-overall",
    "score": 1025.9,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Imagen 4 Ultra' (Google); Overall ranking #15, Overall ELO 1025.9 (aggregated across Alignment 1057, Preference 1106, Coherence 869, Text coherence 1081). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "seedream-5-0-lite",
    "bench": "rapidata-benchai-image-overall",
    "score": 1013.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Seedream 5.0 Lite' (ByteDance); Overall ranking #16, Overall ELO 1013.7 (aggregated across Alignment 1027, Preference 896, Coherence 957, Text coherence 1183). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "flux-2-flex",
    "bench": "rapidata-benchai-image-overall",
    "score": 1011,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'FLUX.2 flex' (Black Forest Labs); Overall ranking #17, Overall ELO 1011.0 (aggregated across Alignment 1034, Preference 1036, Coherence 872, Text coherence 1104). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "grok-imagine-image-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 1009.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Grok Imagine Image (quality)' (SpaceXAI); Overall ranking #18, Overall ELO 1009.7 (aggregated across Alignment 1139, Preference 1034, Coherence 880, Text coherence 980); xAI API model grok-imagine-image-quality, which xAI aliases as grok-imagine-image-pro (the catalogue entry's API); snapshot date not stated. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "wan-2-7-image-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 1007.6,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Wan 2.7 Pro' (Alibaba Cloud); Overall ranking #19, Overall ELO 1007.6 (aggregated across Alignment 1018, Preference 1038, Coherence 987, Text coherence 981); Alibaba Cloud's Wan 2.7 Pro in a text-to-image benchmark, i.e. wan2.7-image-pro. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "flux-2-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 1003.4,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'FLUX.2 pro' (Black Forest Labs); Overall ranking #20, Overall ELO 1003.4 (aggregated across Alignment 1015, Preference 946, Coherence 990, Text coherence 1061). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "seedream-5-0-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 992.9,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Seedream 5.0 Pro' (ByteDance); Overall ranking #21, Overall ELO 992.9 (aggregated across Alignment 1005, Preference 1076, Coherence 1016, Text coherence 858). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "flux-2-max",
    "bench": "rapidata-benchai-image-overall",
    "score": 992.8,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'FLUX.2 max' (Black Forest Labs); Overall ranking #22, Overall ELO 992.8 (aggregated across Alignment 992, Preference 932, Coherence 975, Text coherence 1071). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "qwen-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 991.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Qwen-Image' (Alibaba Cloud); Overall ranking #23, Overall ELO 991.7 (aggregated across Alignment 933, Preference 1129, Coherence 1003, Text coherence 888). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "grok-imagine-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 988.6,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Grok Imagine Image' (SpaceXAI); Overall ranking #24, Overall ELO 988.6 (aggregated across Alignment 1038, Preference 1016, Coherence 866, Text coherence 1031); xAI API model grok-imagine-image. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "hidream-o1-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 988.2,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'HiDream O1' (HiDream AI); Overall ranking #25, Overall ELO 988.2 (aggregated across Alignment 1043, Preference 1101, Coherence 871, Text coherence 928); HiDream AI's HiDream-O1 image model. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "qwen-image-2-0-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 986.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Qwen-Image 2.0 Pro' (Alibaba Cloud); Overall ranking #26, Overall ELO 986.3 (aggregated across Alignment 1005, Preference 920, Coherence 939, Text coherence 1087); release label retained, no API snapshot inferred (same mapping as existing Qwen-Image-Bench/FEPBench rows). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "ideogram-4-5",
    "bench": "rapidata-benchai-image-overall",
    "score": 967.6,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Ideogram 4.5 Quality' (Ideogram); Overall ranking #27, Overall ELO 967.6 (aggregated across Alignment 851, Preference 821, Coherence 1136, Text coherence 1006); Ideogram 4.5 at the Quality rendering tier. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "recraft-v4-1-utility-pro",
    "bench": "rapidata-benchai-image-overall",
    "score": 953.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Recraft V4.1 Utility Pro' (Recraft); Overall ranking #28, Overall ELO 953.3 (aggregated across Alignment 983, Preference 890, Coherence 1004, Text coherence 925). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "flux-2-dev",
    "bench": "rapidata-benchai-image-overall",
    "score": 945.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'FLUX.2 dev' (Black Forest Labs); Overall ranking #29, Overall ELO 945.7 (aggregated across Alignment 1016, Preference 936, Coherence 911, Text coherence 910). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "imagen-4",
    "bench": "rapidata-benchai-image-overall",
    "score": 934.7,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Imagen 4' (Google); Overall ranking #31, Overall ELO 934.7 (aggregated across Alignment 879, Preference 1036, Coherence 901, Text coherence 903); plain Imagen 4, listed separately from Imagen 4 Fast and Imagen 4 Ultra. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "rapidata-benchai-image-overall",
    "score": 925.3,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'FLUX.2 klein 9B' (Black Forest Labs); Overall ranking #33, Overall ELO 925.3 (aggregated across Alignment 950, Preference 905, Coherence 992, Text coherence 837). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "rapidata-benchai-image-overall",
    "score": 924.8,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'HunyuanImage 3.0' (Tencent); Overall ranking #34, Overall ELO 924.8 (aggregated across Alignment 989, Preference 1130, Coherence 767, Text coherence 788). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "ideogram-4",
    "bench": "rapidata-benchai-image-overall",
    "score": 911.9,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Ideogram 4.0 Quality' (Ideogram); Overall ranking #35, Overall ELO 911.9 (aggregated across Alignment 817, Preference 800, Coherence 1100, Text coherence 911); Ideogram 4.0 at the Quality tier (the LMArena row for ideogram-4 is also the 'ideogram-4.0-quality' entry). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "ernie-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 896.9,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'ERNIE Image' (Baidu); Overall ranking #36, Overall ELO 896.9 (aggregated across Alignment 860, Preference 1145, Coherence 694, Text coherence 846). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "p-image",
    "bench": "rapidata-benchai-image-overall",
    "score": 890.2,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Pruna P-Image' (Pruna AI); Overall ranking #38, Overall ELO 890.2 (aggregated across Alignment 634, Preference 874, Coherence 1130, Text coherence 884); Pruna AI's P-Image. 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "uni-1-1-max",
    "bench": "rapidata-benchai-image-overall",
    "score": 846.1,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Luma UNI 1.1 Max' (Luma Labs); Overall ranking #41, Overall ELO 846.1 (aggregated across Alignment 820, Preference 619, Coherence 1014, Text coherence 872). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "sd-3-5-large",
    "bench": "rapidata-benchai-image-overall",
    "score": 794.4,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Stable Diffusion 3.5 Large' (Stability AI); Overall ranking #43, Overall ELO 794.4 (aggregated across Alignment 780, Preference 868, Coherence 862, Text coherence 627). 1,514 prompts, human pairwise votes.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "seedream-3-0",
    "bench": "rapidata-face-alignment",
    "score": 1123.63,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'bytedance/seedream-3' (Replicate model id); leaderboard score 1123.630000 (2604 wins / 4472 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "qwen-image",
    "bench": "rapidata-face-alignment",
    "score": 1110.54,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'qwen/qwen-image' (Replicate model id); leaderboard score 1110.540000 (1654 wins / 2877 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "recraft-v3",
    "bench": "rapidata-face-alignment",
    "score": 1105.07,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'recraft-ai/recraft-v3' (Replicate model id); leaderboard score 1105.070000 (1157 wins / 2028 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "flux-1-1-pro",
    "bench": "rapidata-face-alignment",
    "score": 969.56,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'black-forest-labs/flux-1.1-pro' (Replicate model id); leaderboard score 969.560000 (1272 wins / 2650 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "ideogram-3",
    "bench": "rapidata-face-alignment",
    "score": 962.89,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'ideogram-ai/ideogram-v3-balanced' (Replicate model id); leaderboard score 962.890000 (2868 wins / 6170 matches); Ideogram 3.0 at the Balanced speed tier (the LMArena ideogram-3 row is the 'ideogram-v3-quality' tier). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "luma-photon",
    "bench": "rapidata-face-alignment",
    "score": 955.45,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'luma/photon' (Replicate model id); leaderboard score 955.450000 (1839 wins / 3934 matches); Luma Photon (not Photon Flash, which is a separate row). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "flux-1-schnell",
    "bench": "rapidata-face-alignment",
    "score": 885.06,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'black-forest-labs/flux-schnell' (Replicate model id); leaderboard score 885.060000 (1024 wins / 2417 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "seedance-2-0",
    "bench": "videoargus-t2v",
    "score": 0.934,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Seedance 2.0' (–) in the T2V leaderboard; Table 2 VideoArgus score 9.34/10 -> /10; API model (Seedance 2.0, ref [33]).",
    "source_id": "visual-videoargus"
  },
  {
    "model": "veo-3-1",
    "bench": "videoargus-t2v",
    "score": 0.866,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Veo 3.1' (–) in the T2V leaderboard; Table 2 VideoArgus score 8.66/10 -> /10; API model Veo 3.1 (ref [7]).",
    "source_id": "visual-videoargus"
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "videoargus-t2v",
    "score": 0.854,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'HunyuanVideo-1.5' (8.3B) in the T2V leaderboard; Table 2 VideoArgus score 8.54/10 -> /10; HunyuanVideo-1.5, 8.3B (ref [46]) in T2V mode.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "videoargus-t2v",
    "score": 0.841,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Wan2.2-T2V' (14B) in the T2V leaderboard; Table 2 VideoArgus score 8.41/10 -> /10; Wan2.2-T2V, 14B (ref [39]) = Wan2.2-T2V-A14B.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "longcat-video",
    "bench": "videoargus-t2v",
    "score": 0.811,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'LongCat-Video' (13.6B) in the T2V leaderboard; Table 2 VideoArgus score 8.11/10 -> /10; LongCat-Video, 13.6B (ref [38]).",
    "source_id": "visual-videoargus"
  },
  {
    "model": "cogvideox-1-5-5b",
    "bench": "videoargus-t2v",
    "score": 0.752,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'CogVideoX1.5' (5B) in the T2V leaderboard; Table 2 VideoArgus score 7.52/10 -> /10; CogVideoX1.5, 5B (ref [50]) in T2V mode.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "cogvideox-5b",
    "bench": "videoargus-t2v",
    "score": 0.739,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'CogVideoX' (5B) in the T2V leaderboard; Table 2 VideoArgus score 7.39/10 -> /10; CogVideoX, 5B (ref [50]) in T2V mode.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "seedance-2-0",
    "bench": "videoargus-ti2v",
    "score": 0.897,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Seedance 2.0' (–) in the TI2V leaderboard; Table 2 VideoArgus score 8.97/10 -> /10; API model (Seedance 2.0, ref [33]).",
    "source_id": "visual-videoargus"
  },
  {
    "model": "veo-3-1",
    "bench": "videoargus-ti2v",
    "score": 0.836,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Veo 3.1' (–) in the TI2V leaderboard; Table 2 VideoArgus score 8.36/10 -> /10; API model Veo 3.1 (ref [7]).",
    "source_id": "visual-videoargus"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "videoargus-ti2v",
    "score": 0.811,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'HunyuanVideo-1.5' (8.3B) in the TI2V leaderboard; Table 2 VideoArgus score 8.11/10 -> /10; HunyuanVideo-1.5, 8.3B (ref [46]) in text+image-to-video mode.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "videoargus-ti2v",
    "score": 0.782,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Wan2.2-TI2V' (5B) in the TI2V leaderboard; Table 2 VideoArgus score 7.82/10 -> /10; Wan2.2-TI2V, 5B (ref [39]) = Wan2.2-TI2V-5B.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "cogvideox-5b-i2v",
    "bench": "videoargus-ti2v",
    "score": 0.729,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'CogVideoX' (5B) in the TI2V leaderboard; Table 2 VideoArgus score 7.29/10 -> /10; CogVideoX, 5B (ref [50]) in the TI2V task; the only CogVideoX 1.0 5B image-to-video release is CogVideoX-5b-I2V.",
    "source_id": "visual-videoargus"
  },
  {
    "model": "seedance-2-5",
    "bench": "datapoint-video-bench-elo",
    "score": 1049.3,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Seedance 2.5'; Datapoint Video Bench overall Elo 1049.3 (rank 1, 95% rank spread 1–4). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "datapoint-video-bench-elo",
    "score": 1049.3,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Gemini Omni 1.1 Flash'; Datapoint Video Bench overall Elo 1049.3 (rank 2, 95% rank spread 1–4). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "wan-3-0",
    "bench": "datapoint-video-bench-elo",
    "score": 1045.9,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Wan 3.0'; Datapoint Video Bench overall Elo 1045.9 (rank 3, 95% rank spread 1–5). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "flux-3-video",
    "bench": "datapoint-video-bench-elo",
    "score": 1039.2,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'FLUX.3 Video'; Datapoint Video Bench overall Elo 1039.2 (rank 4, 95% rank spread 1–6); BFL FLUX 3 Video. Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "kling-3-0-pro",
    "bench": "datapoint-video-bench-elo",
    "score": 1010.4,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Kling 3.0 Pro'; Datapoint Video Bench overall Elo 1010.4 (rank 7, 95% rank spread 6–10). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "happyhorse-1-1",
    "bench": "datapoint-video-bench-elo",
    "score": 998.6,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'HappyHorse 1.1'; Datapoint Video Bench overall Elo 998.6 (rank 8, 95% rank spread 7–10). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "grok-imagine-video-1-5",
    "bench": "datapoint-video-bench-elo",
    "score": 980.3,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Grok Imagine 1.5'; Datapoint Video Bench overall Elo 980.3 (rank 11, 95% rank spread 9–12); xAI Grok Imagine (video) 1.5 in a text-to-video benchmark. Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "ltx-2-5",
    "bench": "datapoint-video-bench-elo",
    "score": 978.4,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'LTX 2.5 Pro'; Datapoint Video Bench overall Elo 978.4 (rank 12, 95% rank spread 11–13); LTX-2.5 via the LTX API pro tier (the catalogue entry's API model is ltx-2-5-pro). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "veo-3-1",
    "bench": "datapoint-video-bench-elo",
    "score": 963.8,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Veo 3.1'; Datapoint Video Bench overall Elo 963.8 (rank 13, 95% rank spread 12–14). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "vidu-q3-pro",
    "bench": "datapoint-video-bench-elo",
    "score": 953.5,
    "source": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md#leaderboard",
    "by": "independent",
    "date": "2026-09-28",
    "note": "'Vidu Q3 Pro'; Datapoint Video Bench overall Elo 953.5 (rank 14, 95% rank spread 13–14). Human pairwise votes, 314 prompts.",
    "source_id": "visual-datapoint-video-bench"
  },
  {
    "model": "deepfloyd-if-xl",
    "bench": "evalalign-human-alignment",
    "score": 5.45,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'IF-I-XL v1.0 (Alex Shonenkov & et al., 2023)'; Table 3 text-to-image alignment, Human column 5.4500 (cell '5.45001{}^{1~~}' = value then rank superscript); DeepFloyd IF-I-XL v1.0. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "deepfloyd-if-l",
    "bench": "evalalign-human-alignment",
    "score": 5.23,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'IF-I-L v1.0 (Alex Shonenkov & et al., 2023)'; Table 3 text-to-image alignment, Human column 5.2300 (cell '5.23002{}^{2~~}' = value then rank superscript); DeepFloyd IF-I-L v1.0. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "lcm-sdxl",
    "bench": "evalalign-human-alignment",
    "score": 5.18,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'LCM SDXL (Luo et al., 2023)'; Table 3 text-to-image alignment, Human column 5.1800 (cell '5.18004{}^{4~~}' = value then rank superscript); LCM SDXL (Luo et al. 2023, listed separately from LCM-LoRA SDXL), i.e. the full latent-consistency/lcm-sdxl distillation. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "pixart-alpha",
    "bench": "evalalign-human-alignment",
    "score": 5.11,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'PixArt XL2 1024 MS (Chen et al., 2023b)'; Table 3 text-to-image alignment, Human column 5.1100 (cell '5.11005{}^{5~~}' = value then rank superscript); PixArt-XL-2-1024-MS. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "deepfloyd-if-m",
    "bench": "evalalign-human-alignment",
    "score": 5.08,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'IF-I-M v1.0 (Alex Shonenkov & et al., 2023)'; Table 3 text-to-image alignment, Human column 5.0800 (cell '5.08006{}^{6~~}' = value then rank superscript); DeepFloyd IF-I-M v1.0. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sdxl",
    "bench": "evalalign-human-alignment",
    "score": 5.03,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'SDXL v1.0 (Podell et al., 2023)'; Table 3 text-to-image alignment, Human column 5.0300 (cell '5.03008{}^{8~~}' = value then rank superscript); SDXL base 1.0 (the SDXL Refiner row is separate). 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "openjourney",
    "bench": "evalalign-human-alignment",
    "score": 4.83,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'Openjourney (PromptHero, a)'; Table 3 text-to-image alignment, Human column 4.8300 (cell '4.830010' = value then rank superscript); reference: huggingface.co/prompthero/openjourney (v1). 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sd-2-1",
    "bench": "evalalign-human-alignment",
    "score": 4.8,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'SD v2.1 (Rombach et al., 2022a)'; Table 3 text-to-image alignment, Human column 4.8000 (cell '4.800011' = value then rank superscript); release-level identity, as for the catalogue entry. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sd-2-0",
    "bench": "evalalign-human-alignment",
    "score": 4.64,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'SD v2.0 (Rombach et al., 2022a)'; Table 3 text-to-image alignment, Human column 4.6400 (cell '4.640014' = value then rank superscript); release-level identity, as for the catalogue entry. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "dreamlike-photoreal-2-0",
    "bench": "evalalign-human-alignment",
    "score": 4.56,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'Dreamlike Photoreal v2.0 (dreamlike.art, b)'; Table 3 text-to-image alignment, Human column 4.5600 (cell '4.560017' = value then rank superscript); reference: huggingface.co/dreamlike-art/dreamlike-photoreal-2.0. 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sd-1-4",
    "bench": "evalalign-human-alignment",
    "score": 4.52,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'SD v1.4 (Rombach et al., 2022a)'; Table 3 text-to-image alignment, Human column 4.5200 (cell '4.520019' = value then rank superscript). 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sd-1-5",
    "bench": "evalalign-human-alignment",
    "score": 4.45,
    "source": "https://arxiv.org/html/2406.16562v3#S5.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "'SD v1.5 (Rombach et al., 2022a)'; Table 3 text-to-image alignment, Human column 4.4500 (cell '4.450020' = value then rank superscript). 100 prompts, human annotation.",
    "source_id": "visual-evalalign"
  },
  {
    "model": "sdxl",
    "bench": "coco10k-fid-sdxl-lightning",
    "score": 18.49,
    "source": "https://arxiv.org/html/2402.13929v3#S4.T2",
    "by": "independent",
    "date": "2024-03-02",
    "note": "'SDXL [44]', 32 steps; Table 2 FID (Whole) 18.49; SDXL 1.0 base, 32 steps.",
    "source_id": "visual-sdxl-lightning"
  },
  {
    "model": "lcm-sdxl",
    "bench": "coco10k-fid-sdxl-lightning",
    "score": 21.85,
    "source": "https://arxiv.org/html/2402.13929v3#S4.T2",
    "by": "independent",
    "date": "2024-03-02",
    "note": "'LCM [36]', 4 steps; Table 2 FID (Whole) 21.85; LCM (full UNet distillation of SDXL, listed separately from LCM-LoRA), 4 steps.",
    "source_id": "visual-sdxl-lightning"
  },
  {
    "model": "sdxl-turbo",
    "bench": "coco10k-fid-sdxl-lightning",
    "score": 23.71,
    "source": "https://arxiv.org/html/2402.13929v3#S4.T2",
    "by": "independent",
    "date": "2024-03-02",
    "note": "'Turbo [58]', 1 steps; Table 2 FID (Whole) 23.71; SDXL-Turbo at its designed 1 step (the 4-step row is not imported).",
    "source_id": "visual-sdxl-lightning"
  },
  {
    "model": "sdxl-lightning-4-step",
    "bench": "coco10k-fid-sdxl-lightning",
    "score": 22.3,
    "source": "https://arxiv.org/html/2402.13929v3#S4.T2",
    "by": "lab",
    "date": "2024-03-02",
    "note": "'Ours', 4 steps; Table 2 FID (Whole) 22.30; SDXL-Lightning full-UNet 4-step checkpoint (not the LoRA rows); evaluator is the model's maker.",
    "source_id": "visual-sdxl-lightning"
  },
  {
    "model": "cosmos3-super-text2image",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.9136,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "lab",
    "date": "2026-06-23",
    "note": "'Cosmos3-Super-Text2Image'; Table 11 UniGenBench All (1,170 prompts) 91.36 -> 91.36/100; NVIDIA's own model, huggingface.co/nvidia/Cosmos3-Super-Text2Image.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "nano-banana-pro",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.9069,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "'Gemini 3 Pro Image'; Table 11 UniGenBench All (1,170 prompts) 90.69 -> 90.69/100; closed-source API.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "flux-2-dev",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.876,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "'FLUX.2-dev'; Table 11 UniGenBench All (1,170 prompts) 87.60 -> 87.60/100; open weights.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "qwen-image-2512",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.8425,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "'Qwen-Image-2512'; Table 11 UniGenBench All (1,170 prompts) 84.25 -> 84.25/100; open weights.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.8402,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "'Hunyuan 3.0'; Table 11 UniGenBench All (1,170 prompts) 84.02 -> 84.02/100; open weights, HunyuanImage 3.0.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "z-image-turbo",
    "bench": "unigenbench-phys-all-cosmos3",
    "score": 0.7814,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "'Z-Image-Turbo'; Table 11 UniGenBench All (1,170 prompts) 78.14 -> 78.14/100; open weights.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "veo-3-1",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.595,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Veo3.1'; Table 1 MindWorldBench Intention Accuracy 59.5% -> 59.5/100; Veo 3.1 (Google blog, accessed Aug 2026).",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.293,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Seedance1.5Pro'; Table 1 MindWorldBench Intention Accuracy 29.3% -> 29.3/100; Seedance 1.5 Pro.",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "runway-gen-4",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.186,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Gen4'; Table 1 MindWorldBench Intention Accuracy 18.6% -> 18.6/100; Runway Gen-4 (reference: 'Introducing Runway Gen-4', accessed 2026-04-02).",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "hailuo-2-3",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.13,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Hailuo2.3'; Table 1 MindWorldBench Intention Accuracy 13.0% -> 13.0/100; MiniMax Hailuo 2.3 (hailuoai.video, accessed 2026-04-02).",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "cogvideox-5b-i2v",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.277,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'CogVideoX'; Table 1 MindWorldBench Intention Accuracy 27.7% -> 27.7/100; CogVideoX 5B in an image-to-video benchmark = CogVideoX-5b-I2V.",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "wan-2-6",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.39,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Wan2.6'; Table 1 MindWorldBench Intention Accuracy 39.0% -> 39.0/100; Wan 2.6 (I2V).",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.287,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'Wan2.2'; Table 1 MindWorldBench Intention Accuracy 28.7% -> 28.7/100; Wan2.2 14B in an image-to-video benchmark = Wan2.2-I2V-A14B.",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.23,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'HunyuanVideo1.5'; Table 1 MindWorldBench Intention Accuracy 23.0% -> 23.0/100; HunyuanVideo 1.5 (8B) in I2V mode.",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "longcat-video",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.222,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'LongCat-Video'; Table 1 MindWorldBench Intention Accuracy 22.2% -> 22.2/100; LongCat-Video 13.6B.",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "ltx-2-3",
    "bench": "mindworldbench-intention-accuracy",
    "score": 0.119,
    "source": "https://arxiv.org/html/2609.39147v1#S3.T1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "'LTX2.3'; Table 1 MindWorldBench Intention Accuracy 11.9% -> 11.9/100; LTX-2.3 22B (model card huggingface.co/Lightricks/LTX-2.3).",
    "source_id": "visual-mindworldbench"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.389,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'FLUX.1-Kontext-dev (Labs, 2025b)'; Table 3 Result Metric Avg 38.9% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "flux-2-dev",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.299,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'FLUX.2-dev (Labs, 2025a)'; Table 3 Result Metric Avg 29.9% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.322,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Qwen-Image-Edit-2509 (Wu et al., 2025a)'; Table 3 Result Metric Avg 32.2% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.258,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Qwen-Image-Edit-2511 (Wu et al., 2025a)'; Table 3 Result Metric Avg 25.8% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "longcat-image-edit",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.322,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "lab",
    "date": "2026-03-26",
    "note": "'LongCat-Image-Edit (Ma et al., 2025)'; Table 3 Result Metric Avg 32.2% -> /100; Meituan co-authors the report, so marked lab. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "bagel",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.231,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Bagel (Deng et al., 2025)'; Table 3 Result Metric Avg 23.1% -> /100; BAGEL without thinking (the Bagel-Think CoT row is separate). Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "omnigen2",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.224,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'OmniGen2 (Wu et al., 2025b)'; Table 3 Result Metric Avg 22.4% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "uniworld-v1",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.29,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'UniWorld-V1 (Lin et al., 2025)'; Table 3 Result Metric Avg 29.0% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "lumina-dimoo",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.34,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'DiMOO (Xin et al., 2025)'; Table 3 Result Metric Avg 34.0% -> /100; Lumina-DiMOO (Xin et al., 2025). Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "seedream-4-0",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.415,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Seedream 4.0 (Seedream et al., 2025a)'; Table 3 Result Metric Avg 41.5% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "gpt-image-1",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.469,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'GPT-image-1 (OpenAI, 2025a)'; Table 3 Result Metric Avg 46.9% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "nano-banana",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.477,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Nano Banana (Google, 2025b)'; Table 3 Result Metric Avg 47.7% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "nano-banana-pro",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.684,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Nano Banana Pro (Google, 2025c)'; Table 3 Result Metric Avg 68.4% -> /100. Without-CoT block.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "seedance-1-0-pro",
    "bench": "vigor-bench-result-avg-video",
    "score": 0.296,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Seedance 1.0 Pro (Gao et al., 2025)'; Table 3 Result Metric Avg 29.6% -> /100. Video generator, final frame scored.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "veo-3",
    "bench": "vigor-bench-result-avg-video",
    "score": 0.205,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Veo 3 (Google, 2024)'; Table 3 Result Metric Avg 20.5% -> /100; reference 'Google, 2024' as printed. Video generator, final frame scored.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "sora-2-pro",
    "bench": "vigor-bench-result-avg-video",
    "score": 0.178,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Sora 2 Pro (OpenAI, 2025c)'; Table 3 Result Metric Avg 17.8% -> /100. Video generator, final frame scored.",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "phyground-human-overall",
    "score": 0.656,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'Wan2.2-27B-A14B'; Table 2 PhyGround human Overall 3.28 on 1-5 -> 3.28/5; Wan2.2 27B-A14B run in the ti2v format, i.e. the Wan2.2-I2V-A14B checkpoint.",
    "source_id": "visual-phyground"
  },
  {
    "model": "veo-3-1",
    "bench": "phyground-human-overall",
    "score": 0.656,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'Veo-3.1†'; Table 2 PhyGround human Overall 3.28 on 1-5 -> 3.28/5; closed-source API.",
    "source_id": "visual-phyground"
  },
  {
    "model": "ltx-2-3",
    "bench": "phyground-human-overall",
    "score": 0.538,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'LTX-2.3-22B'; Table 2 PhyGround human Overall 2.69 on 1-5 -> 2.69/5; LTX-2.3 22B.",
    "source_id": "visual-phyground"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "phyground-human-overall",
    "score": 0.526,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'Wan2.2-TI2V-5B'; Table 2 PhyGround human Overall 2.63 on 1-5 -> 2.63/5; Wan2.2-TI2V-5B.",
    "source_id": "visual-phyground"
  },
  {
    "model": "ltx-2",
    "bench": "phyground-human-overall",
    "score": 0.512,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'LTX-2-19B'; Table 2 PhyGround human Overall 2.56 on 1-5 -> 2.56/5; LTX-2 19B (not the distilled checkpoint).",
    "source_id": "visual-phyground"
  },
  {
    "model": "seedream-4-0",
    "bench": "edival-o-turn3",
    "score": 0.5976,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'Seedream 4.0' (release date column 25.09.10); Table 3 EdiVal-O at turn 3 59.76 -> /100 (overall rank 1).",
    "source_id": "visual-edival"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "edival-o-turn3",
    "score": 0.5955,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'GPT-Image-1.5' (release date column 25.12.16); Table 3 EdiVal-O at turn 3 59.55 -> /100 (overall rank 2).",
    "source_id": "visual-edival"
  },
  {
    "model": "nano-banana-2",
    "bench": "edival-o-turn3",
    "score": 0.5849,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'Nano Banana 2' (release date column 26.03.01); Table 3 EdiVal-O at turn 3 58.49 -> /100 (overall rank 3).",
    "source_id": "visual-edival"
  },
  {
    "model": "nano-banana",
    "bench": "edival-o-turn3",
    "score": 0.5624,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'Nano Banana' (release date column 25.08.26); Table 3 EdiVal-O at turn 3 56.24 -> /100 (overall rank 5).",
    "source_id": "visual-edival"
  },
  {
    "model": "gpt-image-1",
    "bench": "edival-o-turn3",
    "score": 0.5381,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'GPT-Image-1' (release date column 25.07.16); Table 3 EdiVal-O at turn 3 53.81 -> /100 (overall rank 6).",
    "source_id": "visual-edival"
  },
  {
    "model": "flux-2-max",
    "bench": "edival-o-turn3",
    "score": 0.581,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'FLUX.2-max' (release date column 25.12.16); Table 3 EdiVal-O at turn 3 58.10 -> /100 (overall rank 4).",
    "source_id": "visual-edival"
  },
  {
    "model": "flux-1-kontext-max",
    "bench": "edival-o-turn3",
    "score": 0.5304,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'FLUX.1-Kontext-max' (release date column 25.06.03); Table 3 EdiVal-O at turn 3 53.04 -> /100 (overall rank 7).",
    "source_id": "visual-edival"
  },
  {
    "model": "qwen-image-edit",
    "bench": "edival-o-turn3",
    "score": 0.4193,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'Qwen-Image-Edit' (release date column 25.08.04); Table 3 EdiVal-O at turn 3 41.93 -> /100 (overall rank 9).",
    "source_id": "visual-edival"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "edival-o-turn3",
    "score": 0.3871,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'FLUX.1-Kontext-dev' (release date column 25.06.25); Table 3 EdiVal-O at turn 3 38.71 -> /100 (overall rank 11).",
    "source_id": "visual-edival"
  },
  {
    "model": "instructpix2pix",
    "bench": "edival-o-turn3",
    "score": 0.1299,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'IP2P' (release date column 23.12.15); Table 3 EdiVal-O at turn 3 12.99 -> /100 (overall rank 16).",
    "source_id": "visual-edival"
  },
  {
    "model": "step1x-edit-v1-1",
    "bench": "weavebench-avg",
    "score": 0.669,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'Step1X-Edit v1.1 [48]'; Table 2 WEAVEBench Avg 0.669 (0-1).",
    "source_id": "visual-weavebench"
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "weavebench-avg",
    "score": 0.605,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'Step1X-Edit v1.2 [48]'; Table 2 WEAVEBench Avg 0.605 (0-1).",
    "source_id": "visual-weavebench"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "weavebench-avg",
    "score": 0.689,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'FLUX.1 Kontext [42]'; Table 2 WEAVEBench Avg 0.689 (0-1); FLUX.1 Kontext, 12B open model = Kontext [dev].",
    "source_id": "visual-weavebench"
  },
  {
    "model": "qwen-image-edit",
    "bench": "weavebench-avg",
    "score": 0.665,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'Qwen-Image-Edit [76]'; Table 2 WEAVEBench Avg 0.665 (0-1).",
    "source_id": "visual-weavebench"
  },
  {
    "model": "omnigen2",
    "bench": "weavebench-avg",
    "score": 0.609,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'OminiGen2 [77]'; Table 2 WEAVEBench Avg 0.609 (0-1); printed 'OminiGen2' (typo), 7B, ref [77] OmniGen2.",
    "source_id": "visual-weavebench"
  },
  {
    "model": "bagel",
    "bench": "weavebench-avg",
    "score": 0.446,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'Bagel [22]'; Table 2 WEAVEBench Avg 0.446 (0-1); Bagel 14B (BAGEL-7B-MoT, 14B total), not the Bagel-Zebra or WEAVE-100k fine-tunes.",
    "source_id": "visual-weavebench"
  },
  {
    "model": "flux-2-max",
    "bench": "knowvis-brief-overall-en",
    "score": 0.3498,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'FLUX.2-maxBlack Forest Labs (2025)'; Table 2 (Brief Caption) Overall, English column 34.98 -> 34.98/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "seedream-4-0",
    "bench": "knowvis-brief-overall-en",
    "score": 0.3961,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'Seedream-4.0Seedream et al. (2025)'; Table 2 (Brief Caption) Overall, English column 39.61 -> 39.61/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "nano-banana-pro",
    "bench": "knowvis-brief-overall-en",
    "score": 0.344,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'gemini-3-pro-imageGoogle DeepMind (2026)'; Table 2 (Brief Caption) Overall, English column 34.40 -> 34.40/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "flux-1-krea-dev",
    "bench": "knowvis-brief-overall-en",
    "score": 0.1819,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'FLUX.1-Krea-devLee et al. (2025a)'; Table 2 (Brief Caption) Overall, English column 18.19 -> 18.19/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "sd-3-5-large",
    "bench": "knowvis-brief-overall-en",
    "score": 0.3107,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'SD3.5-Largecalcuis (2024)'; Table 2 (Brief Caption) Overall, English column 31.07 -> 31.07/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "bagel",
    "bench": "knowvis-brief-overall-en",
    "score": 0.222,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'BAGEL-7B-MoT Deng et al. (2025a)'; Table 2 (Brief Caption) Overall, English column 22.20 -> 22.20/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "flux-1-dev",
    "bench": "knowvis-brief-overall-en",
    "score": 0.3393,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'FLUX.1-dev Labs (2024)'; Table 2 (Brief Caption) Overall, English column 33.93 -> 33.93/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "omnigen2",
    "bench": "knowvis-brief-overall-en",
    "score": 0.1847,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'OmniGen2Wu et al. (2025c)'; Table 2 (Brief Caption) Overall, English column 18.47 -> 18.47/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "flux-2-dev",
    "bench": "knowvis-brief-overall-en",
    "score": 0.3688,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'FLUX.2-devBlack Forest Labs (2025)'; Table 2 (Brief Caption) Overall, English column 36.88 -> 36.88/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "qwen-image",
    "bench": "knowvis-brief-overall-en",
    "score": 0.2574,
    "source": "https://arxiv.org/html/2604.22302v1#S4.T2",
    "by": "independent",
    "date": "2026-04-24",
    "note": "'Qwen-ImageWu et al. (2025b)'; Table 2 (Brief Caption) Overall, English column 25.74 -> 25.74/100 (Chinese column not imported).",
    "source_id": "visual-knowledge-visualization"
  },
  {
    "model": "bagel",
    "bench": "realpref-bench-human",
    "score": 0.7176,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'BAGEL[5]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 3.588 -> 3.588/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "uniworld-v1",
    "bench": "realpref-bench-human",
    "score": 0.6526,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'UniWorld-V1[19]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 3.263 -> 3.263/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "omnigen2",
    "bench": "realpref-bench-human",
    "score": 0.687,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'Omnigen2[50]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 3.435 -> 3.435/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "qwen-image-edit",
    "bench": "realpref-bench-human",
    "score": 0.801,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'Qwen-Image-Edit[48]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 4.005 -> 4.005/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "realpref-bench-human",
    "score": 0.669,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'FLUX.1 Kontext-Dev[15]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 3.345 -> 3.345/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "realpref-bench-human",
    "score": 0.8674,
    "source": "https://arxiv.org/html/2604.19406v1#S5.T1",
    "by": "independent",
    "date": "2026-04-21",
    "note": "'Qwen-Image-Edit-2509[48]'; Table 1 RealPref-Bench 'Human' (user-study mean, 0-5) 4.337 -> 4.337/5.",
    "source_id": "visual-hp-edit"
  },
  {
    "model": "seedance-2-0",
    "bench": "switch-i2v-vlm-overall",
    "score": 0.528,
    "source": "https://arxiv.org/html/2511.17649v5#S4.T3",
    "by": "independent",
    "date": "2026-07-07",
    "note": "'Doubao-Seed 2.0'; Table 3 Single Image to Video, VLM-as-Judge Overall 2.64 on 1-5 -> /5; Table 7 lists it as 'Seedance 2.0' (10 s, 24 fps).",
    "source_id": "visual-switch-bench"
  },
  {
    "model": "kling-3-0",
    "bench": "switch-i2v-vlm-overall",
    "score": 0.374,
    "source": "https://arxiv.org/html/2511.17649v5#S4.T3",
    "by": "independent",
    "date": "2026-07-07",
    "note": "'Kling 3.0'; Table 3 Single Image to Video, VLM-as-Judge Overall 1.87 on 1-5 -> /5; Table 7 'Kling v3' (10 s, 24 fps); tier not stated, so not mapped to Pro.",
    "source_id": "visual-switch-bench"
  },
  {
    "model": "runway-gen-4-5",
    "bench": "switch-i2v-vlm-overall",
    "score": 0.36,
    "source": "https://arxiv.org/html/2511.17649v5#S4.T3",
    "by": "independent",
    "date": "2026-07-07",
    "note": "'Runway Gen-4.5'; Table 3 Single Image to Video, VLM-as-Judge Overall 1.80 on 1-5 -> /5; 10 s, 24 fps (Table 7).",
    "source_id": "visual-switch-bench"
  },
  {
    "model": "veo-3-1",
    "bench": "switch-i2v-vlm-overall",
    "score": 0.418,
    "source": "https://arxiv.org/html/2511.17649v5#S4.T3",
    "by": "independent",
    "date": "2026-07-07",
    "note": "'Veo 3.1'; Table 3 Single Image to Video, VLM-as-Judge Overall 2.09 on 1-5 -> /5.",
    "source_id": "visual-switch-bench"
  },
  {
    "model": "wan-2-7",
    "bench": "switch-i2v-vlm-overall",
    "score": 0.344,
    "source": "https://arxiv.org/html/2511.17649v5#S4.T3",
    "by": "independent",
    "date": "2026-07-07",
    "note": "'Wan 2.7'; Table 3 Single Image to Video, VLM-as-Judge Overall 1.72 on 1-5 -> /5; single-image-to-video block (10 s, 30 fps).",
    "source_id": "visual-switch-bench"
  },
  {
    "model": "minimax-h3",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1723.6,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'MiniMax H3'; Table 2 human Bradley-Terry Elo 1723.6 (95% CI [1644.5, 1837.6], score rate 77.9%, 52 games); FAL API, reference-to-video.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "seedance-2-5",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1649.6,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Seedance 2.5'; Table 2 human Bradley-Terry Elo 1649.6 (95% CI [1554.2, 1751.5], score rate 69.6%, 51 games); FAL API, reference-to-video.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "gemini-omni-1-1-flash",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1492.5,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Gemini Omni Flash 1.1'; Table 2 human Bradley-Terry Elo 1492.5 (95% CI [1404.1, 1572.9], score rate 49.0%, 52 games); FAL API, frame-wise edit/transfer.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "kling-o3-pro",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1471.3,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Kling O3 Pro'; Table 2 human Bradley-Terry Elo 1471.3 (95% CI [1404.0, 1534.0], score rate 46.2%, 52 games); FAL API, frame-wise edit/transfer.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "gpt-1",
    "bench": "prost-zero-shot-macro",
    "score": 0.261,
    "source": "https://aclanthology.org/2021.findings-acl.404.pdf",
    "by": "independent",
    "date": "2021-08-01",
    "note": "Row GPT (116.5M; 'OpenAI's GPT-1'): 26.1%. Table 5 (printed p. 4603), 'Macro Average' column: mean of the 10 concept accuracies (direction, mass, height and circumference averaged over their templates first). Zero-shot, no fine-tuning; authors ran the Hugging Face pretrained checkpoint (Section 4, Table 4). Percent divided by 100. Date is the Findings of ACL-IJCNLP 2021 publication month.",
    "source_id": "acl-2021-findings-acl-404-prost"
  },
  {
    "model": "gpt-2-small",
    "bench": "prost-zero-shot-macro",
    "score": 0.236,
    "source": "https://aclanthology.org/2021.findings-acl.404.pdf",
    "by": "independent",
    "date": "2021-08-01",
    "note": "Row GPT-2 B (124.4M): 23.6%. Table 5 (printed p. 4603), 'Macro Average' column: mean of the 10 concept accuracies (direction, mass, height and circumference averaged over their templates first). Zero-shot, no fine-tuning; authors ran the Hugging Face pretrained checkpoint (Section 4, Table 4). Percent divided by 100. Date is the Findings of ACL-IJCNLP 2021 publication month.",
    "source_id": "acl-2021-findings-acl-404-prost"
  },
  {
    "model": "gpt-2-medium",
    "bench": "prost-zero-shot-macro",
    "score": 0.268,
    "source": "https://aclanthology.org/2021.findings-acl.404.pdf",
    "by": "independent",
    "date": "2021-08-01",
    "note": "Row GPT-2 M (354.8M): 26.8%. Table 5 (printed p. 4603), 'Macro Average' column: mean of the 10 concept accuracies (direction, mass, height and circumference averaged over their templates first). Zero-shot, no fine-tuning; authors ran the Hugging Face pretrained checkpoint (Section 4, Table 4). Percent divided by 100. Date is the Findings of ACL-IJCNLP 2021 publication month.",
    "source_id": "acl-2021-findings-acl-404-prost"
  },
  {
    "model": "gpt-2-large",
    "bench": "prost-zero-shot-macro",
    "score": 0.264,
    "source": "https://aclanthology.org/2021.findings-acl.404.pdf",
    "by": "independent",
    "date": "2021-08-01",
    "note": "Row GPT-2 L (774.0M): 26.4%. Table 5 (printed p. 4603), 'Macro Average' column: mean of the 10 concept accuracies (direction, mass, height and circumference averaged over their templates first). Zero-shot, no fine-tuning; authors ran the Hugging Face pretrained checkpoint (Section 4, Table 4). Percent divided by 100. Date is the Findings of ACL-IJCNLP 2021 publication month.",
    "source_id": "acl-2021-findings-acl-404-prost"
  },
  {
    "model": "gpt-2-xl",
    "bench": "prost-zero-shot-macro",
    "score": 0.301,
    "source": "https://aclanthology.org/2021.findings-acl.404.pdf",
    "by": "independent",
    "date": "2021-08-01",
    "note": "Row GPT-2 XL (1557.6M): 30.1%. Table 5 (printed p. 4603), 'Macro Average' column: mean of the 10 concept accuracies (direction, mass, height and circumference averaged over their templates first). Zero-shot, no fine-tuning; authors ran the Hugging Face pretrained checkpoint (Section 4, Table 4). Percent divided by 100. Date is the Findings of ACL-IJCNLP 2021 publication month.",
    "source_id": "acl-2021-findings-acl-404-prost"
  },
  {
    "model": "command-a-plus",
    "bench": "tau2-telecom",
    "score": 0.85,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'Compared with Command A Reasoning, τ²-Bench Telecom scores improved from 37% to 85%'. Footnote 1: 'τ²-Bench Telecom evaluated using standard settings and user simulation from [Barres et al. 2025]'. Domain pass rate, no aggregate given. Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-08; figure stated in the post's body text (not read from a chart). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Comparator printed in the same sentence: Command A Reasoning (08-2025) or Command A Vision (07-2025), recorded from the same post.",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-plus",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.25,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'agentic coding performance on Terminal-Bench Hard reaching 25% from 3%' (Command A Reasoning 3%). Footnote 1: 'Terminal-Bench Hard evaluated using Terminus-2 harness following the methodology of Artificial Analysis' (Cohere's own run; not an Artificial Analysis number). Mapped to the catalogue's only 'Terminal Bench Hard' id; review that the Gemma 4 report's TB-Hard is the same subset. Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-08; figure stated in the post's body text (not read from a chart). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Comparator printed in the same sentence: Command A Reasoning (08-2025) or Command A Vision (07-2025), recorded from the same post.",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-plus",
    "bench": "charxiv-rq",
    "score": 0.527,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'CharXiv reasoning improved from 46.9% to 52.7%' (Command A Vision 46.9%). Image 5 caption: 'We follow standard methodology for the given benchmarks.' Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-08; figure stated in the post's body text (not read from a chart). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Comparator printed in the same sentence: Command A Reasoning (08-2025) or Command A Vision (07-2025), recorded from the same post.",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "tau2-telecom",
    "score": 0.37,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'Compared with Command A Reasoning, τ²-Bench Telecom scores improved from 37% to 85%'. Same footnote-1 settings as the Command A+ row. vendor-reported (rule b).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.03,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'Terminal-Bench Hard reaching 25% from 3%' (3% = Command A Reasoning). Terminus-2 harness per footnote 1. vendor-reported (rule b).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-vision-07-2025",
    "bench": "charxiv-rq",
    "score": 0.469,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Text: 'CharXiv reasoning improved from 46.9% to 52.7%' (46.9% = Command A Vision, per Image 5 caption comparing Command A+ with Command A Vision). vendor-reported (rule b).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "bfcl-v4",
    "score": 0.5706,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "note": "Published entry 'Command A Reasoning (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Harness model identifiers: command-a-reasoning-08-2025.",
    "date": "2025-12-16",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "mt-bench-gpt4-single",
    "score": 7.12,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "lab",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'vicuna-33b' ('Vicuna-33B', LMSYS, link https://huggingface.co/lmsys/vicuna-33b-v1.3): 'MT-bench (score)' = 7.12 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'Vicuna-33B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'. LMSYS's own model, so by=lab.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "vicuna-13b-v1-5",
    "bench": "mt-bench-gpt4-single",
    "score": 6.57,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "lab",
    "date": "2023-08-02",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'vicuna-13b' ('Vicuna-13B', LMSYS, link https://huggingface.co/lmsys/vicuna-13b-v1.5): 'MT-bench (score)' = 6.57 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230802 table as 'Vicuna-13B (v1.5 link)' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'. LMSYS's own model, so by=lab. The 2023-06/07 tables carried v1.3 values (6.39/6.00) under the same name; the v1.5 values replaced them on 2023-08-02 when the link changed to the v1.5 repo.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "mt-bench-gpt4-single",
    "score": 6.17,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "lab",
    "date": "2023-08-02",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'vicuna-7b' ('Vicuna-7B', LMSYS, link https://huggingface.co/lmsys/vicuna-7b-v1.5): 'MT-bench (score)' = 6.17 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230802 table as 'Vicuna-7B (v1.5 link)' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'. LMSYS's own model, so by=lab. The 2023-06/07 tables carried v1.3 values (6.39/6.00) under the same name; the v1.5 values replaced them on 2023-08-02 when the link changed to the v1.5 repo.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "fastchat-t5-3b",
    "bench": "mt-bench-gpt4-single",
    "score": 3.04,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "lab",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'fastchat-t5-3b' ('FastChat-T5-3B', LMSYS, link https://huggingface.co/lmsys/fastchat-t5-3b-v1.0): 'MT-bench (score)' = 3.04 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'FastChat-T5-3B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'. LMSYS's own model, so by=lab.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "guanaco-33b",
    "bench": "mt-bench-gpt4-single",
    "score": 6.53,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'guanaco-33b' ('Guanaco-33B', UW, link https://huggingface.co/timdettmers/guanaco-33b-merged): 'MT-bench (score)' = 6.53 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'Guanaco-33B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "mpt-30b-chat",
    "bench": "mt-bench-gpt4-single",
    "score": 6.39,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'mpt-30b-chat' ('MPT-30B-chat', MosaicML, link https://huggingface.co/mosaicml/mpt-30b-chat): 'MT-bench (score)' = 6.39 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'MPT-30B-chat' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "koala-13b",
    "bench": "mt-bench-gpt4-single",
    "score": 5.35,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'koala-13b' ('Koala-13B', UC Berkeley, link https://bair.berkeley.edu/blog/2023/04/03/koala/): 'MT-bench (score)' = 5.35 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'Koala-13B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "mpt-7b-chat",
    "bench": "mt-bench-gpt4-single",
    "score": 5.42,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'mpt-7b-chat' ('MPT-7B-Chat', MosaicML, link https://huggingface.co/mosaicml/mpt-7b-chat): 'MT-bench (score)' = 5.42 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'MPT-7B-Chat' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt4all-13b-snoozy",
    "bench": "mt-bench-gpt4-single",
    "score": 5.41,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt4all-13b-snoozy' ('GPT4All-13B-Snoozy', Nomic AI, link https://huggingface.co/nomic-ai/gpt4all-13b-snoozy): 'MT-bench (score)' = 5.41 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'GPT4All-13B-Snoozy' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "alpaca-13b",
    "bench": "mt-bench-gpt4-single",
    "score": 4.53,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'alpaca-13b' ('Alpaca-13B', Stanford, link https://crfm.stanford.edu/2023/03/13/alpaca.html): 'MT-bench (score)' = 4.53 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'Alpaca-13B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "chatglm-6b",
    "bench": "mt-bench-gpt4-single",
    "score": 4.5,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'chatglm-6b' ('ChatGLM-6B', Tsinghua, link https://huggingface.co/THUDM/chatglm-6b): 'MT-bench (score)' = 4.50 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'ChatGLM-6B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "mt-bench-gpt4-single",
    "score": 4.32,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'oasst-pythia-12b' ('OpenAssistant-Pythia-12B', OpenAssistant, link https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5): 'MT-bench (score)' = 4.32 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'OpenAssistant-Pythia-12B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "rwkv-4-raven-14b",
    "bench": "mt-bench-gpt4-single",
    "score": 3.98,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'RWKV-4-Raven-14B' ('RWKV-4-Raven-14B', RWKV, link https://huggingface.co/BlinkDL/rwkv-4-raven): 'MT-bench (score)' = 3.98 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'RWKV-4-Raven-14B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "dolly-v2-12b",
    "bench": "mt-bench-gpt4-single",
    "score": 3.28,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'dolly-v2-12b' ('Dolly-V2-12B', Databricks, link https://huggingface.co/databricks/dolly-v2-12b): 'MT-bench (score)' = 3.28 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'Dolly-V2-12B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "stablelm-tuned-alpha-7b",
    "bench": "mt-bench-gpt4-single",
    "score": 2.75,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'stablelm-tuned-alpha-7b' ('StableLM-Tuned-Alpha-7B', Stability AI, link https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b): 'MT-bench (score)' = 2.75 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'StableLM-Tuned-Alpha-7B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "llama-13b",
    "bench": "mt-bench-gpt4-single",
    "score": 2.61,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'llama-13b' ('LLaMA-13B', Meta, link https://arxiv.org/abs/2302.13971): 'MT-bench (score)' = 2.61 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'LLaMA-13B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "palm-2-chat-bison",
    "bench": "mt-bench-gpt4-single",
    "score": 6.4,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-06-19",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'palm-2' ('PaLM-Chat-Bison-001', Google, link https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models#foundation_models): 'MT-bench (score)' = 6.40 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230619 table as 'PaLM-Chat-Bison-001' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "mt-bench-gpt4-single",
    "score": 6.86,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-07-17",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'llama-2-70b-chat' ('Llama-2-70B-chat', Meta, link https://huggingface.co/meta-llama/Llama-2-70b-chat-hf): 'MT-bench (score)' = 6.86 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230717 table as 'Llama-2-70b-chat' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "mt-bench-gpt4-single",
    "score": 6.65,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-07-17",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'llama-2-13b-chat' ('Llama-2-13b-chat', Meta, link https://huggingface.co/meta-llama/Llama-2-13b-chat-hf): 'MT-bench (score)' = 6.65 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230717 table as 'Llama-2-13b-chat' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "mt-bench-gpt4-single",
    "score": 6.27,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-07-17",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'llama-2-7b-chat' ('Llama-2-7B-chat', Meta, link https://huggingface.co/meta-llama/Llama-2-7b-chat-hf): 'MT-bench (score)' = 6.27 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230717 table as 'Llama-2-7b-chat' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "chatglm2-6b",
    "bench": "mt-bench-gpt4-single",
    "score": 4.96,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-07-17",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'chatglm2-6b' ('ChatGLM2-6B', Tsinghua, link https://huggingface.co/THUDM/chatglm2-6b): 'MT-bench (score)' = 4.96 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230717 table as 'ChatGLM2-6B' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "claude-2-0",
    "bench": "mt-bench-gpt4-single",
    "score": 8.06,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-07-17",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'claude-2.0' ('Claude-2.0', Anthropic, link https://www.anthropic.com/index/claude-2): 'MT-bench (score)' = 8.06 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20230717 table as 'Claude-2 (relabelled Claude-2.0 on 2023-12-06)' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "mt-bench-gpt4-single",
    "score": 9.32,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-11-16",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt-4-1106-preview' ('GPT-4-1106-preview', OpenAI, link https://openai.com/blog/new-models-and-developer-products-announced-at-devday): 'MT-bench (score)' = 9.32 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231116 table as 'GPT-4-Turbo' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt-4-0314",
    "bench": "mt-bench-gpt4-single",
    "score": 8.96,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-12-06",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt-4-0314' ('GPT-4-0314', OpenAI, link https://openai.com/research/gpt-4): 'MT-bench (score)' = 8.96 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231206 table as 'GPT-4-0314' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt-4-0613",
    "bench": "mt-bench-gpt4-single",
    "score": 9.18,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-12-06",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt-4-0613' ('GPT-4-0613', OpenAI, link https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo): 'MT-bench (score)' = 9.18 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231206 table as 'GPT-4-0613' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "mt-bench-gpt4-single",
    "score": 8.39,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-12-06",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt-3.5-turbo-0613' ('GPT-3.5-Turbo-0613', OpenAI, link https://platform.openai.com/docs/models/gpt-3-5): 'MT-bench (score)' = 8.39 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231206 table as 'GPT-3.5-turbo-0613' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "mt-bench-gpt4-single",
    "score": 8.32,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-12-06",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'gpt-3.5-turbo-1106' ('GPT-3.5-Turbo-1106', OpenAI, link https://platform.openai.com/docs/models/gpt-3-5): 'MT-bench (score)' = 8.32 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231206 table as 'GPT-3.5-Turbo-1106' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "claude-2-1",
    "bench": "mt-bench-gpt4-single",
    "score": 8.18,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2023-12-06",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'claude-2.1' ('Claude-2.1', Anthropic, link https://www.anthropic.com/index/claude-2-1): 'MT-bench (score)' = 8.18 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). Value first published in the 20231206 table as 'Claude-2.1' and unchanged since. FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'.",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "mmlu",
    "score": 0.697,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU' (metric '5-shot'), column 'Jamba-1.5 Mini' = 69.7. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "mmlu",
    "score": 0.8,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU' (metric '5-shot'), column 'Jamba-1.5 Large' = 80.0. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "mmlu",
    "score": 0.825,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU' (metric '5-shot'), column 'Mistral-L-2 123B' = 82.5, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "mmlu-pro",
    "score": 0.398,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU Pro' (metric '5-shot'), column 'Jamba-1.5 Mini' = 39.8. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "mmlu-pro",
    "score": 0.483,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU Pro' (metric '5-shot'), column 'Jamba-1.5 Large' = 48.3. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "mmlu-pro",
    "score": 0.542,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'MMLU Pro' (metric '5-shot'), column 'Mistral-L-2 123B' = 54.2, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "gpqa-unspecified",
    "score": 0.323,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GPQA' (metric '0-shot'), column 'Jamba-1.5 Mini' = 32.3. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "gpqa-unspecified",
    "score": 0.369,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GPQA' (metric '0-shot'), column 'Jamba-1.5 Large' = 36.9. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "gpqa-unspecified",
    "score": 0.407,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GPQA' (metric '0-shot'), column 'Mistral-L-2 123B' = 40.7, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "arc-challenge",
    "score": 0.857,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'ARC-C' (metric '0-shot'), column 'Jamba-1.5 Mini' = 85.7. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "arc-challenge",
    "score": 0.93,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'ARC-C' (metric '0-shot'), column 'Jamba-1.5 Large' = 93.0. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "arc-challenge",
    "score": 0.65,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'ARC-C' (metric '0-shot'), column 'Mistral-L-2 123B' = 65.0, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "bbh",
    "score": 0.534,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BBH' (metric '3-shot'), column 'Jamba-1.5 Mini' = 53.4. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "bbh",
    "score": 0.655,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BBH' (metric '3-shot'), column 'Jamba-1.5 Large' = 65.5. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "bbh",
    "score": 0.708,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BBH' (metric '3-shot'), column 'Mistral-L-2 123B' = 70.8, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "humaneval",
    "score": 0.628,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'HumanEval' (metric 'pass@1'), column 'Jamba-1.5 Mini' = 62.8. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "humaneval",
    "score": 0.713,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'HumanEval' (metric 'pass@1'), column 'Jamba-1.5 Large' = 71.3. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "gsm8k",
    "score": 0.758,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GSM8K' (metric '5-shot'), column 'Jamba-1.5 Mini' = 75.8. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "gsm8k",
    "score": 0.87,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GSM8K' (metric '5-shot'), column 'Jamba-1.5 Large' = 87.0. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "gsm8k",
    "score": 0.91,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'GSM8K' (metric '5-shot'), column 'Mistral-L-2 123B' = 91.0, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "ifeval-unspecified",
    "score": 0.758,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'IFEval' (metric '0-shot'), column 'Jamba-1.5 Mini' = 75.8. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "ifeval-unspecified",
    "score": 0.815,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'IFEval' (metric '0-shot'), column 'Jamba-1.5 Large' = 81.5. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'IFEval' (metric '0-shot'), column 'Mistral-L-2 123B' = 87.8, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "bfcl-version-unspecified",
    "score": 0.807,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BFCL' (metric '0-shot'), column 'Jamba-1.5 Mini' = 80.7. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "bfcl-version-unspecified",
    "score": 0.855,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BFCL' (metric '0-shot'), column 'Jamba-1.5 Large' = 85.5. AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "mistral-large-2407",
    "bench": "bfcl-version-unspecified",
    "score": 0.851,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T2",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 2, row 'BFCL' (metric '0-shot'), column 'Mistral-L-2 123B' = 85.1, marked '† evaluation run by us' (AI21 ran it). Mistral-L-2 (123B) in an August 2024 paper = Mistral-Large-Instruct-2407. Unmarked and ⋄ (Open LLM Leaderboard) competitor cells were not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "arena-hard-unspecified",
    "score": 0.461,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T3",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 3, row 'Arena-Hard', column 'Jamba-1.5 Mini' = 46.1 (version/judge not stated; Aug 2024 = Arena-Hard-Auto v0.1 era). AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "jamba-1-5-large",
    "bench": "arena-hard-unspecified",
    "score": 0.654,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T3",
    "by": "lab",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 3, row 'Arena-Hard', column 'Jamba-1.5 Large' = 65.4 (version/judge not stated; Aug 2024 = Arena-Hard-Auto v0.1 era). AI21's own model.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "gemma-2-9b",
    "bench": "arena-hard-unspecified",
    "score": 0.432,
    "source": "https://arxiv.org/html/2408.12570v1#S6.T3",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Jamba-1.5 paper (AI21, arXiv 2408.12570v1, CC BY-SA 4.0), Table 3, row 'Arena-Hard', column 'Gemma-2 9B' = 43.2, marked '† evaluation run by us' (AI21 ran it; instruction-tuned chat model, as in the chatbot comparison). Other competitor Arena-Hard cells are unmarked (reported) and not used.",
    "source_id": "paper-2408.12570v1-jamba15"
  },
  {
    "model": "pixtral-12b",
    "bench": "mathvista-mini",
    "score": 0.583,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Pixtral 12B', column 'Mathvista (CoT)' = 58.3. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "mmmu",
    "score": 0.52,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Pixtral 12B', column 'MMMU (CoT)' = 52.0. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "chartqa",
    "score": 0.818,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Pixtral 12B', column 'ChartQA (CoT)' = 81.8. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "vqav2-val",
    "score": 0.786,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Pixtral 12B', column 'VQAv2 (VQA Match)' = 78.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "mm-mt-bench",
    "score": 6.05,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Pixtral 12B', column 'MM-MT-Bench (GPT-4o Judge)' = 6.05. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mathvista-mini",
    "score": 0.537,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 7B', column 'Mathvista (CoT)' = 53.7. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mmmu",
    "score": 0.481,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 7B', column 'MMMU (CoT)' = 48.1. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "chartqa",
    "score": 0.412,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 7B', column 'ChartQA (CoT)' = 41.2. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "vqav2-val",
    "score": 0.759,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 7B', column 'VQAv2 (VQA Match)' = 75.9. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mm-mt-bench",
    "score": 5.45,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 7B', column 'MM-MT-Bench (GPT-4o Judge)' = 5.45. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "mathvista-mini",
    "score": 0.243,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 11B', column 'Mathvista (CoT)' = 24.3. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "mmmu",
    "score": 0.23,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 11B', column 'MMMU (CoT)' = 23.0. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "chartqa",
    "score": 0.148,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 11B', column 'ChartQA (CoT)' = 14.8. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "vqav2-val",
    "score": 0.671,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 11B', column 'VQAv2 (VQA Match)' = 67.1. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "mm-mt-bench",
    "score": 4.79,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 11B', column 'MM-MT-Bench (GPT-4o Judge)' = 4.79. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mathvista-mini",
    "score": 0.123,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo-D 7B', column 'Mathvista (CoT)' = 12.3. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mmmu",
    "score": 0.243,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo-D 7B', column 'MMMU (CoT)' = 24.3. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "chartqa",
    "score": 0.27,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo-D 7B', column 'ChartQA (CoT)' = 27.0. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "vqav2-val",
    "score": 0.571,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo-D 7B', column 'VQAv2 (VQA Match)' = 57.1. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mm-mt-bench",
    "score": 3.72,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo-D 7B', column 'MM-MT-Bench (GPT-4o Judge)' = 3.72. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mathvista-mini",
    "score": 0.448,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3 Haiku', column 'Mathvista (CoT)' = 44.8. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Claude 3 Haiku (only snapshot 20240307).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mmmu",
    "score": 0.504,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3 Haiku', column 'MMMU (CoT)' = 50.4. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Claude 3 Haiku (only snapshot 20240307).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "chartqa",
    "score": 0.696,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3 Haiku', column 'ChartQA (CoT)' = 69.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Claude 3 Haiku (only snapshot 20240307).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "vqav2-val",
    "score": 0.684,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3 Haiku', column 'VQAv2 (VQA Match)' = 68.4. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Claude 3 Haiku (only snapshot 20240307).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mm-mt-bench",
    "score": 5.46,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3 Haiku', column 'MM-MT-Bench (GPT-4o Judge)' = 5.46. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Claude 3 Haiku (only snapshot 20240307).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-72b",
    "bench": "mathvista-mini",
    "score": 0.522,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo 72B', column 'Mathvista (CoT)' = 52.2. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-72B-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-72b",
    "bench": "mmmu",
    "score": 0.527,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo 72B', column 'MMMU (CoT)' = 52.7. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-72B-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-72b",
    "bench": "chartqa",
    "score": 0.756,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo 72B', column 'ChartQA (CoT)' = 75.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-72B-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-72b",
    "bench": "vqav2-val",
    "score": 0.752,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo 72B', column 'VQAv2 (VQA Match)' = 75.2. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-72B-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-72b",
    "bench": "mm-mt-bench",
    "score": 3.51,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Molmo 72B', column 'MM-MT-Bench (GPT-4o Judge)' = 3.51. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Molmo-72B-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mathvista-mini",
    "score": 0.682,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 72B', column 'Mathvista (CoT)' = 68.2. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-72B-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmmu",
    "score": 0.603,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 72B', column 'MMMU (CoT)' = 60.3. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-72B-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "chartqa",
    "score": 0.666,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 72B', column 'ChartQA (CoT)' = 66.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-72B-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "vqav2-val",
    "score": 0.816,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 72B', column 'VQAv2 (VQA Match)' = 81.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-72B-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mm-mt-bench",
    "score": 6.59,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Qwen-2-VL 72B', column 'MM-MT-Bench (GPT-4o Judge)' = 6.59. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Qwen2-VL-72B-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "mathvista-mini",
    "score": 0.491,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 90B', column 'Mathvista (CoT)' = 49.1. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-90B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "mmmu",
    "score": 0.537,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 90B', column 'MMMU (CoT)' = 53.7. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-90B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "chartqa",
    "score": 0.338,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 90B', column 'ChartQA (CoT)' = 33.8. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-90B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "vqav2-val",
    "score": 0.67,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 90B', column 'VQAv2 (VQA Match)' = 67.0. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-90B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "mm-mt-bench",
    "score": 5.5,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Llama-3.2 90B', column 'MM-MT-Bench (GPT-4o Judge)' = 5.5. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Llama-3.2-90B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mathvista-mini",
    "score": 0.646,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'GPT-4o (0513)', column 'Mathvista (CoT)' = 64.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mmmu",
    "score": 0.686,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'GPT-4o (0513)', column 'MMMU (CoT)' = 68.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "chartqa",
    "score": 0.851,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'GPT-4o (0513)', column 'ChartQA (CoT)' = 85.1. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "vqav2-val",
    "score": 0.778,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'GPT-4o (0513)', column 'VQAv2 (VQA Match)' = 77.8. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mm-mt-bench",
    "score": 7.72,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'GPT-4o (0513)', column 'MM-MT-Bench (GPT-4o Judge)' = 7.72. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mathvista-mini",
    "score": 0.644,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3.5 Sonnet', column 'Mathvista (CoT)' = 64.4. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Paper v1/v2 dated 2024-10-09/10, before the 2024-10-22 Sonnet, so the June 2024 snapshot.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mmmu",
    "score": 0.68,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3.5 Sonnet', column 'MMMU (CoT)' = 68.0. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Paper v1/v2 dated 2024-10-09/10, before the 2024-10-22 Sonnet, so the June 2024 snapshot.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "chartqa",
    "score": 0.876,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3.5 Sonnet', column 'ChartQA (CoT)' = 87.6. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Paper v1/v2 dated 2024-10-09/10, before the 2024-10-22 Sonnet, so the June 2024 snapshot.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "vqav2-val",
    "score": 0.707,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3.5 Sonnet', column 'VQAv2 (VQA Match)' = 70.7. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Paper v1/v2 dated 2024-10-09/10, before the 2024-10-22 Sonnet, so the June 2024 snapshot.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mm-mt-bench",
    "score": 7.5,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T2",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 2 (multimodal benchmarks; 'We re-evaluate all models with the same prompt and evaluation metric'), row 'Claude-3.5 Sonnet', column 'MM-MT-Bench (GPT-4o Judge)' = 7.5. Main row (strict 'Explicit' prompt + exact-match parsing), not the grey 'w/ Flexible Parsing' rows. Splits from the cited mistral-evals code: MathVista testmini, MMMU validation, ChartQA test, VQAv2 validation; MM-MT-Bench eval split (92 conversations), GPT-4o judge, 1-10. Paper v1/v2 dated 2024-10-09/10, before the 2024-10-22 Sonnet, so the June 2024 snapshot.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "mmlu",
    "score": 0.692,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Pixtral 12B', column 'MMLU (5-shot)' = 69.2. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "math-subset-unspecified",
    "score": 0.481,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Pixtral 12B', column 'Math (Maj@1)' = 48.1. MATH split not stated. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-12b",
    "bench": "humaneval",
    "score": 0.72,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "lab",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Pixtral 12B', column 'HumanEval (Pass@1)' = 72.0. Mistral's own model.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mmlu",
    "score": 0.612,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Molmo-D 7B', column 'MMLU (5-shot)' = 61.2. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "math-subset-unspecified",
    "score": 0.102,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Molmo-D 7B', column 'Math (Maj@1)' = 10.2. Molmo-7B-D-0924. MATH split not stated.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "molmo-7b-d",
    "bench": "humaneval",
    "score": 0.037,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Molmo-D 7B', column 'HumanEval (Pass@1)' = 3.7. Molmo-7B-D-0924.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mmlu",
    "score": 0.685,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Qwen-2-VL 7B', column 'MMLU (5-shot)' = 68.5. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "math-subset-unspecified",
    "score": 0.279,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Qwen-2-VL 7B', column 'Math (Maj@1)' = 27.9. Qwen2-VL-7B-Instruct (chat model evaluated with prompts). MATH split not stated.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "humaneval",
    "score": 0.622,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Qwen-2-VL 7B', column 'HumanEval (Pass@1)' = 62.2. Qwen2-VL-7B-Instruct (chat model evaluated with prompts).",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "mmlu",
    "score": 0.685,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Llama-3.2 11B', column 'MMLU (5-shot)' = 68.5. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "math-subset-unspecified",
    "score": 0.483,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Llama-3.2 11B', column 'Math (Maj@1)' = 48.3. Llama-3.2-11B-Vision-Instruct. MATH split not stated.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "llama-3-2-11b-vision",
    "bench": "humaneval",
    "score": 0.628,
    "source": "https://arxiv.org/html/2410.07073v2#S3.T3",
    "by": "independent",
    "date": "2024-10-10",
    "note": "Pixtral 12B paper (Mistral AI, arXiv 2410.07073v2, CC BY 4.0), Table 3 (language benchmarks; text says 'again, with common prompting and evaluation protocols'), row 'Llama-3.2 11B', column 'HumanEval (Pass@1)' = 62.8. Llama-3.2-11B-Vision-Instruct.",
    "source_id": "paper-2410.07073v2-pixtral"
  },
  {
    "model": "pixtral-large",
    "bench": "mathvista-mini",
    "score": 0.694,
    "source": "https://huggingface.co/mistralai/Pixtral-Large-Instruct-2411/blob/c1e51f6f11974a1199685d35c62f5a425c2d001e/README.md#metrics",
    "by": "lab",
    "date": "2024-11-18",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Pixtral-Large-Instruct-2411 card, section 'Metrics', row 'Pixtral Large (124B)', column 'MathVista (CoT)' = 69.4. Mistral's evaluation harness (mistral-evals linked from the card: MathVista testmini, MMMU validation, ChartQA test). Competitor rows ('measured' by Mistral) not used: the card is under the Mistral Research License.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "pixtral-large",
    "bench": "mmmu",
    "score": 0.64,
    "source": "https://huggingface.co/mistralai/Pixtral-Large-Instruct-2411/blob/c1e51f6f11974a1199685d35c62f5a425c2d001e/README.md#metrics",
    "by": "lab",
    "date": "2024-11-18",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Pixtral-Large-Instruct-2411 card, section 'Metrics', row 'Pixtral Large (124B)', column 'MMMU (CoT)' = 64.0. Mistral's evaluation harness (mistral-evals linked from the card: MathVista testmini, MMMU validation, ChartQA test). Competitor rows ('measured' by Mistral) not used: the card is under the Mistral Research License.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "pixtral-large",
    "bench": "chartqa",
    "score": 0.881,
    "source": "https://huggingface.co/mistralai/Pixtral-Large-Instruct-2411/blob/c1e51f6f11974a1199685d35c62f5a425c2d001e/README.md#metrics",
    "by": "lab",
    "date": "2024-11-18",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Pixtral-Large-Instruct-2411 card, section 'Metrics', row 'Pixtral Large (124B)', column 'ChartQA (CoT)' = 88.1. Mistral's evaluation harness (mistral-evals linked from the card: MathVista testmini, MMMU validation, ChartQA test). Competitor rows ('measured' by Mistral) not used: the card is under the Mistral Research License.",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mathvista-mini",
    "score": 0.6891,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/blob/68faf511d618ef198fef186659617cfd2eb8e33a/README.md#vision",
    "by": "lab",
    "date": "2025-03-17",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Mistral-Small-3.1-24B-Instruct-2503 card (Apache-2.0), 'Instruction Evals > Vision', row 'Small 3.1 24B Instruct', column 'Mathvista' = 68.91%. Competitor rows not used (card: 'When available, we report numbers previously published by other model providers, otherwise we re-evaluate them').",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mmmu",
    "score": 0.64,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/blob/68faf511d618ef198fef186659617cfd2eb8e33a/README.md#vision",
    "by": "lab",
    "date": "2025-03-17",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Mistral-Small-3.1-24B-Instruct-2503 card (Apache-2.0), 'Instruction Evals > Vision', row 'Small 3.1 24B Instruct', column 'MMMU' = 64.0%. Competitor rows not used (card: 'When available, we report numbers previously published by other model providers, otherwise we re-evaluate them').",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "chartqa",
    "score": 0.8624,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/blob/68faf511d618ef198fef186659617cfd2eb8e33a/README.md#vision",
    "by": "lab",
    "date": "2025-03-17",
    "note": "Vendor-reported (Mistral, own model), hand-recorded from the pinned Mistral-Small-3.1-24B-Instruct-2503 card (Apache-2.0), 'Instruction Evals > Vision', row 'Small 3.1 24B Instruct', column 'ChartQA' = 86.24%. Competitor rows not used (card: 'When available, we report numbers previously published by other model providers, otherwise we re-evaluate them').",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "molmo-72b",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1077,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "lab",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Molmo 72B (this model)', column 'Human Preference Elo Rating' = 1077. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Ai2's own model.",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "molmo-7b-d",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1056,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "lab",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Molmo 7B-D', column 'Human Preference Elo Rating' = 1056. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Ai2's own model.",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1069,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Claude 3.5 Sonnet', column 'Human Preference Elo Rating' = 1069. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Only Claude 3.5 Sonnet snapshot in Sept 2024.",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "molmo-human-pref-elo-2409",
    "score": 971,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Claude 3 Opus', column 'Human Preference Elo Rating' = 971. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Only Claude 3 Opus snapshot.",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "molmo-human-pref-elo-2409",
    "score": 999,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Claude 3 Haiku', column 'Human Preference Elo Rating' = 999. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Only Claude 3 Haiku snapshot.",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1037,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Qwen VL2 72B', column 'Human Preference Elo Rating' = 1037. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Qwen2-VL-72B-Instruct (the chat model; only instruct weights released).",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1025,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Qwen VL2 7B', column 'Human Preference Elo Rating' = 1025. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Qwen2-VL-7B-Instruct (chat model).",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "pixtral-12b",
    "bench": "molmo-human-pref-elo-2409",
    "score": 1016,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Pixtral 12B', column 'Human Preference Elo Rating' = 1016. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Pixtral-12B-2409 (released 2024-09-11).",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "molmo-human-pref-elo-2409",
    "score": 982,
    "source": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations",
    "by": "independent",
    "date": "2024-09-25",
    "note": "Molmo-72B-0924 model card (Ai2, Apache-2.0) @ab586dd, 'Evaluations' table, row 'Phi3.5-Vision 4B', column 'Human Preference Elo Rating' = 982. Ai2's own human evaluation (Molmo paper arXiv 2409.17146 Sec. 'Human evaluation': Ai2 queried each VLM on ~15k image-text prompts and collected ~325k pairwise human preference judgments across 27 models; Elo via Bradley-Terry). Card first published 2024-09-25; table unchanged since. Phi-3.5-vision-instruct (4.2B).",
    "source_id": "molmo-card-human-elo"
  },
  {
    "model": "gpt-5",
    "bench": "molmo2-human-pref-elo",
    "score": 1031,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5', column 'Elo Score' = 1031. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7). GPT-5 API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gpt-5",
    "bench": "molmo2-captest-f1",
    "score": 0.501,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5', column 'Molmo2 Caption test F1 Score' = 50.1. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. GPT-5 API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gpt-5",
    "bench": "molmo2-videocount-acc",
    "score": 0.358,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5', column 'Molmo2 Count val accuracy' = 35.8. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. GPT-5 API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gpt-5-mini",
    "bench": "molmo2-human-pref-elo",
    "score": 1076,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5 mini', column 'Elo Score' = 1076. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7). GPT-5 mini API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gpt-5-mini",
    "bench": "molmo2-captest-f1",
    "score": 0.566,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5 mini', column 'Molmo2 Caption test F1 Score' = 56.6. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. GPT-5 mini API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gpt-5-mini",
    "bench": "molmo2-videocount-acc",
    "score": 0.298,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'GPT-5 mini', column 'Molmo2 Count val accuracy' = 29.8. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. GPT-5 mini API (reasoning effort not stated).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "molmo2-human-pref-elo",
    "score": 1082,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 3 Pro', column 'Elo Score' = 1082. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7). Gemini 3 Pro was available only as gemini-3-pro-preview when the study ran (v1 Jan 2026).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "molmo2-captest-f1",
    "score": 0.36,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 3 Pro', column 'Molmo2 Caption test F1 Score' = 36.0. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. Gemini 3 Pro was available only as gemini-3-pro-preview when the study ran (v1 Jan 2026).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "molmo2-videocount-acc",
    "score": 0.371,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 3 Pro', column 'Molmo2 Count val accuracy' = 37.1. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. Gemini 3 Pro was available only as gemini-3-pro-preview when the study ran (v1 Jan 2026).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "molmo2-human-pref-elo",
    "score": 1096,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Pro', column 'Elo Score' = 1096. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "molmo2-captest-f1",
    "score": 0.421,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Pro', column 'Molmo2 Caption test F1 Score' = 42.1. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "molmo2-videocount-acc",
    "score": 0.358,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Pro', column 'Molmo2 Count val accuracy' = 35.8. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "molmo2-human-pref-elo",
    "score": 1084,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Flash', column 'Elo Score' = 1084. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "molmo2-captest-f1",
    "score": 0.46,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Flash', column 'Molmo2 Caption test F1 Score' = 46.0. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "molmo2-videocount-acc",
    "score": 0.319,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Gemini 2.5 Flash', column 'Molmo2 Count val accuracy' = 31.9. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "molmo2-human-pref-elo",
    "score": 1008,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Claude Sonnet 4.5', column 'Elo Score' = 1008. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "molmo2-captest-f1",
    "score": 0.26,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Claude Sonnet 4.5', column 'Molmo2 Caption test F1 Score' = 26.0. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "molmo2-videocount-acc",
    "score": 0.272,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "independent",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Claude Sonnet 4.5', column 'Molmo2 Count val accuracy' = 27.2. New Ai2 benchmark introduced in this report, so every row is Ai2's own run.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "molmo2-human-pref-elo",
    "score": 1057,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Molmo2-8B', column 'Elo Score' = 1057. Ai2's own human preference study (Sec. 4.1: 450 open-ended video questions + 51 captioning videos, two model outputs per pair, over 105K ratings, 501 per model pair, Bradley-Terry Elo; sampling top_p 0.95, temperature 0.7). Ai2's own model.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "molmo2-captest-f1",
    "score": 0.432,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Molmo2-8B', column 'Molmo2 Caption test F1 Score' = 43.2. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. Ai2's own model.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "molmo2-videocount-acc",
    "score": 0.355,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T2",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 2, row 'Molmo2-8B', column 'Molmo2 Count val accuracy' = 35.5. New Ai2 benchmark introduced in this report, so every row is Ai2's own run. Ai2's own model.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "ai2d-test",
    "score": 0.958,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'AI2D test' = 95.8. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "chartqa",
    "score": 0.86,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'ChartQA test' = 86.0. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "docvqa",
    "score": 0.932,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'DocVQA test' = 93.2. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "infographicvqa",
    "score": 0.801,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'InfoQA test' = 80.1. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "textvqa",
    "score": 0.857,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'TextVQA val' = 85.7. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "vqav2-val",
    "score": 0.87,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'VQA v2.0 val' = 87.0. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "realworldqa",
    "score": 0.776,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'RWQA' = 77.6. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "mmmu",
    "score": 0.53,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'MMMU val' = 53.0. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "mathvista-mini",
    "score": 0.589,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'MathVista testmini' = 58.9. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-2-8b",
    "bench": "countbench",
    "score": 0.937,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo2-8B', column 'CountBench' = 93.7. Ai2's own model, scored by Ai2.",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "ai2d-test",
    "score": 0.932,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'AI2D test' = 93.2. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "chartqa",
    "score": 0.841,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'ChartQA test' = 84.1. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "docvqa",
    "score": 0.922,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'DocVQA test' = 92.2. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "infographicvqa",
    "score": 0.726,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'InfoQA test' = 72.6. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "textvqa",
    "score": 0.817,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'TextVQA val' = 81.7. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "vqav2-val",
    "score": 0.856,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'VQA v2.0 val' = 85.6. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "realworldqa",
    "score": 0.707,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'RWQA' = 70.7. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mmmu",
    "score": 0.453,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'MMMU val' = 45.3. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "mathvista-mini",
    "score": 0.516,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'MathVista testmini' = 51.6. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-7b-d",
    "bench": "countbench",
    "score": 0.885,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-7B-D [29]', column 'CountBench' = 88.5. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "ai2d-test",
    "score": 0.963,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'AI2D test' = 96.3. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "chartqa",
    "score": 0.873,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'ChartQA test' = 87.3. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "docvqa",
    "score": 0.935,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'DocVQA test' = 93.5. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "infographicvqa",
    "score": 0.819,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'InfoQA test' = 81.9. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "textvqa",
    "score": 0.831,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'TextVQA val' = 83.1. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "vqav2-val",
    "score": 0.865,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'VQA v2.0 val' = 86.5. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "realworldqa",
    "score": 0.752,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'RWQA' = 75.2. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "mmmu",
    "score": 0.541,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'MMMU val' = 54.1. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "mathvista-mini",
    "score": 0.586,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'MathVista testmini' = 58.6. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "molmo-72b",
    "bench": "countbench",
    "score": 0.912,
    "source": "https://arxiv.org/html/2601.10611v4#S4.T6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Molmo2 technical report (Ai2, arXiv 2601.10611v4, CC BY 4.0), Table 6, row 'Molmo-72B [29]', column 'CountBench' = 91.2. Ai2's own model, scored by Ai2 (Molmo1 rows cite Ai2's Molmo paper [29]).",
    "source_id": "paper-2601.10611v4-molmo2"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "mmmu",
    "score": 0.404,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "lab",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MMMU (val)', column 'Phi-3-Vision 4.2b' = 40.4. Phi-3-Vision (4.2B) = Phi-3-vision-128k-instruct, the only released Phi-3 vision checkpoint. Microsoft's own model.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mmmu",
    "score": 0.407,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MMMU (val)', column 'Claude 3 haiku' = 40.7. Only Claude 3 Haiku snapshot.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "mmmu",
    "score": 0.555,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MMMU (val)', column 'GPT-4V-Turbo (turbo-2024-04-09)' = 55.5. Column header gives the snapshot turbo-2024-04-09.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "mathvista-mini",
    "score": 0.445,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "lab",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MathVista (testmini)', column 'Phi-3-Vision 4.2b' = 44.5. Phi-3-Vision (4.2B) = Phi-3-vision-128k-instruct, the only released Phi-3 vision checkpoint. Microsoft's own model.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mathvista-mini",
    "score": 0.332,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MathVista (testmini)', column 'Claude 3 haiku' = 33.2. Only Claude 3 Haiku snapshot.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "mathvista-mini",
    "score": 0.475,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'MathVista (testmini)', column 'GPT-4V-Turbo (turbo-2024-04-09)' = 47.5. Column header gives the snapshot turbo-2024-04-09.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "ai2d-test",
    "score": 0.767,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "lab",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'AI2D (test)', column 'Phi-3-Vision 4.2b' = 76.7. Phi-3-Vision (4.2B) = Phi-3-vision-128k-instruct, the only released Phi-3 vision checkpoint. Microsoft's own model.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "ai2d-test",
    "score": 0.603,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'AI2D (test)', column 'Claude 3 haiku' = 60.3. Only Claude 3 Haiku snapshot.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "ai2d-test",
    "score": 0.747,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'AI2D (test)', column 'GPT-4V-Turbo (turbo-2024-04-09)' = 74.7. Column header gives the snapshot turbo-2024-04-09.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "chartqa",
    "score": 0.814,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "lab",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'ChartQA (test)', column 'Phi-3-Vision 4.2b' = 81.4. Phi-3-Vision (4.2B) = Phi-3-vision-128k-instruct, the only released Phi-3 vision checkpoint. Microsoft's own model.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "chartqa",
    "score": 0.593,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'ChartQA (test)', column 'Claude 3 haiku' = 59.3. Only Claude 3 Haiku snapshot.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "chartqa",
    "score": 0.623,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'ChartQA (test)', column 'GPT-4V-Turbo (turbo-2024-04-09)' = 62.3. Column header gives the snapshot turbo-2024-04-09.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "textvqa",
    "score": 0.709,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "lab",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'TextVQA (test)', column 'Phi-3-Vision 4.2b' = 70.9. Phi-3-Vision (4.2B) = Phi-3-vision-128k-instruct, the only released Phi-3 vision checkpoint. Microsoft's own model.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "textvqa",
    "score": 0.627,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'TextVQA (test)', column 'Claude 3 haiku' = 62.7. Only Claude 3 Haiku snapshot.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "textvqa",
    "score": 0.681,
    "source": "https://arxiv.org/html/2404.14219v3#S7",
    "by": "independent",
    "date": "2024-05-23",
    "note": "Phi-3 Technical Report v3 (Microsoft, arXiv 2404.14219v3, 23 May 2024, CC BY 4.0), Table 2 ('All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat and MM1-7B-Chat'; LLaVA-1.5 setting, 0-shot), row 'TextVQA (test)', column 'GPT-4V-Turbo (turbo-2024-04-09)' = 68.1. Column header gives the snapshot turbo-2024-04-09.",
    "source_id": "paper-2404.14219v3-phi3"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mmmu",
    "score": 0.43,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MMMU (val)', column 'Phi-3.5-vision-instruct' = 43.0. Microsoft's own model.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "internvl2-4b",
    "bench": "mmmu",
    "score": 0.4422,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MMMU (val)', column 'Intern-VL-2-4B' = 44.22. OpenGVLab InternVL2-4B.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmmu",
    "score": 0.521,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MMMU (val)', column 'GPT-4o-mini 2024-7-18' = 52.1.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mmmu",
    "score": 0.5267,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MMMU (val)', column 'Claude-3.5-Sonnet' = 52.67. Card published Aug 2024, when 20240620 was the only Claude 3.5 Sonnet.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mmmu",
    "score": 0.6178,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MMMU (val)', column 'GPT-4o 2024-5-13' = 61.78.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mathvista-mini",
    "score": 0.439,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MathVista (testmini)', column 'Phi-3.5-vision-instruct' = 43.9. Microsoft's own model.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "internvl2-4b",
    "bench": "mathvista-mini",
    "score": 0.537,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MathVista (testmini)', column 'Intern-VL-2-4B' = 53.7. OpenGVLab InternVL2-4B.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mathvista-mini",
    "score": 0.388,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MathVista (testmini)', column 'GPT-4o-mini 2024-7-18' = 38.8.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mathvista-mini",
    "score": 0.54,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MathVista (testmini)', column 'Claude-3.5-Sonnet' = 54.0. Card published Aug 2024, when 20240620 was the only Claude 3.5 Sonnet.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mathvista-mini",
    "score": 0.544,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'MathVista (testmini)', column 'GPT-4o 2024-5-13' = 54.4.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "ai2d-test",
    "score": 0.781,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'AI2D (test)', column 'Phi-3.5-vision-instruct' = 78.1. Microsoft's own model.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "internvl2-4b",
    "bench": "ai2d-test",
    "score": 0.773,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'AI2D (test)', column 'Intern-VL-2-4B' = 77.3. OpenGVLab InternVL2-4B.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "ai2d-test",
    "score": 0.752,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'AI2D (test)', column 'GPT-4o-mini 2024-7-18' = 75.2.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "ai2d-test",
    "score": 0.689,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'AI2D (test)', column 'Claude-3.5-Sonnet' = 68.9. Card published Aug 2024, when 20240620 was the only Claude 3.5 Sonnet.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "ai2d-test",
    "score": 0.828,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'AI2D (test)', column 'GPT-4o 2024-5-13' = 82.8.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "chartqa",
    "score": 0.818,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'ChartQA (test)', column 'Phi-3.5-vision-instruct' = 81.8. Microsoft's own model.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "internvl2-4b",
    "bench": "chartqa",
    "score": 0.788,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'ChartQA (test)', column 'Intern-VL-2-4B' = 78.8. OpenGVLab InternVL2-4B.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "chartqa",
    "score": 0.545,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'ChartQA (test)', column 'GPT-4o-mini 2024-7-18' = 54.5.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "chartqa",
    "score": 0.732,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'ChartQA (test)', column 'Claude-3.5-Sonnet' = 73.2. Card published Aug 2024, when 20240620 was the only Claude 3.5 Sonnet.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "chartqa",
    "score": 0.64,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'ChartQA (test)', column 'GPT-4o 2024-5-13' = 64.0.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "textvqa",
    "score": 0.72,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "lab",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'TextVQA (val)', column 'Phi-3.5-vision-instruct' = 72.0. Microsoft's own model.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "internvl2-4b",
    "bench": "textvqa",
    "score": 0.662,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'TextVQA (val)', column 'Intern-VL-2-4B' = 66.2. OpenGVLab InternVL2-4B.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "textvqa",
    "score": 0.709,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'TextVQA (val)', column 'GPT-4o-mini 2024-7-18' = 70.9.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "textvqa",
    "score": 0.705,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'TextVQA (val)', column 'Claude-3.5-Sonnet' = 70.5. Card published Aug 2024, when 20240620 was the only Claude 3.5 Sonnet.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "textvqa",
    "score": 0.756,
    "source": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/12b77fb40b63a2c73c68243d3f767aab688a1b2a/README.md#benchmarks",
    "by": "independent",
    "date": "2024-08-20",
    "note": "Phi-3.5-vision-instruct model card (Microsoft, MIT) @12b77fb, 'Benchmarks' table ('we compare Phi-3.5-vision with a set of models over a variety of zero-shot benchmarks using our internal benchmark platform'), row 'TextVQA (val)', column 'GPT-4o 2024-5-13' = 75.6.",
    "source_id": "microsoft-phi35-vision-card-benchmarks"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "gpqa-diamond",
    "score": 0.667,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "aime-2024",
    "score": 0.675,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "aime-2025",
    "score": 0.6,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "math-500",
    "score": 0.966,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "bfcl-v2-live",
    "score": 0.737,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "livecodebench-202408-202502",
    "score": 0.455,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "ifeval-unspecified",
    "score": 0.892,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run. The report specifies IFEval (Strict-Instruction); kept on ifeval-unspecified to match the existing LN-Ultra row (both could move to ifeval-instruction-strict).",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1",
    "bench": "arena-hard-unspecified",
    "score": 0.883,
    "source": "https://arxiv.org/html/2505.00949v5",
    "by": "lab",
    "date": "2025-09-09",
    "note": "NVIDIA's own evaluation, Table 4 'LN-Super Reasoning' column = Llama-3.3-Nemotron-Super-49B-v1 (listed by name in the report's release list; not the SFT-only checkpoint, not v1.5). Reasoning ON value (first of the on | off pair); temperature 0.6, top-p 0.95, up to 16 samples, average pass@1, 32k context. Same report and bench ids as the existing LN-Ultra rows, so both models now share each run. Arena-Hard version/judge not stated.",
    "source_id": "report-2505.00949v5"
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "math-500",
    "score": 0.828,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210/blob/6f134cbe88cb9284a8ce696e8ac8eefd0bc24ede/README.md#1-introduction",
    "by": "lab",
    "date": "2024-12-10",
    "note": "Vendor-reported (DeepSeek, own model), hand-recorded from the pinned DeepSeek-V2.5-1210 card. Model card 'Introduction': 'Mathematical: Performance on the MATH-500 benchmark has improved from 74.8% to 82.8%.' 82.8% = DeepSeek-V2.5-1210.",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "math-500",
    "score": 0.748,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210/blob/6f134cbe88cb9284a8ce696e8ac8eefd0bc24ede/README.md#1-introduction",
    "by": "lab",
    "date": "2024-12-10",
    "note": "Vendor-reported (DeepSeek, own model), hand-recorded from the pinned DeepSeek-V2.5-1210 card. Model card 'Introduction': 'Mathematical: Performance on the MATH-500 benchmark has improved from 74.8% to 82.8%.' 74.8% = the previous DeepSeek-V2.5 (linked in the same sentence as https://huggingface.co/deepseek-ai/DeepSeek-V2.5, the 0905 release).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "llama-3-8b",
    "bench": "bfcl-version-unspecified",
    "score": 0.5888,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "independent",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'Llama-3-8B-Instruct', column 'Overall' = 58.88.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "bfcl-version-unspecified",
    "score": 0.8124,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "independent",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'GPT-4 Turbo (2024-04-09)', column 'Overall' = 81.24.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "bfcl-version-unspecified",
    "score": 0.8294,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "independent",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'GPT-4o (2024-05-13)', column 'Overall' = 82.94.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "chatglm3-6b",
    "bench": "bfcl-version-unspecified",
    "score": 0.5788,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'ChatGLM3-6B', column 'Overall' = 57.88. Zhipu/Tsinghua's own model.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "bfcl-version-unspecified",
    "score": 0.81,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'GLM-4-9B-Chat', column 'Overall' = 81.0. Zhipu/Tsinghua's own model.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "bfcl-version-unspecified",
    "score": 0.8176,
    "source": "https://arxiv.org/html/2406.12793v2#S3.T7",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM, arXiv 2406.12793v2, 30 Jul 2024, CC BY 4.0), Table 7 'GLM performance on the Berkeley Function Call Leaderboard' (Sec. 3.6: 'we carry out evaluations on Berkeley Function Call Leaderboard, a benchmark with 2k question-function-answer pairs', i.e. the original BFCL; version not named), row 'GLM-4 (0520)', column 'Overall' = 81.76. Zhipu/Tsinghua's own model.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "llada-image",
    "bench": "longtext-en",
    "score": 0.923,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T5",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 5 (long-text rendering), row 'LLaDA-Image', column 'LongText-Bench-EN' = 0.923. Own-model lab result (competitor rows not used); same report and ledger entry as the existing GenEval/DPG rows.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "llada-image",
    "bench": "longtext-cn",
    "score": 0.913,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T5",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 5 (long-text rendering), row 'LLaDA-Image', column 'LongText-Bench-ZH' = 0.913. Own-model lab result (competitor rows not used); same report and ledger entry as the existing GenEval/DPG rows.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "llada-image",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.875,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T6",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 6 (CVTG-2K), row 'LLaDA-Image', Word Accuracy 'average' column = 0.875 (2/3/4/5 regions and average: 0.892 | 0.878 | 0.882 | 0.857 | 0.875); NED and CLIPScore not used. Own-model lab result.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "llada-image-turbo",
    "bench": "longtext-en",
    "score": 0.899,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T5",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 5 (long-text rendering), row 'LLaDA-Image Turbo', column 'LongText-Bench-EN' = 0.899. Own-model lab result (competitor rows not used); same report and ledger entry as the existing GenEval/DPG rows.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "llada-image-turbo",
    "bench": "longtext-cn",
    "score": 0.919,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T5",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 5 (long-text rendering), row 'LLaDA-Image Turbo', column 'LongText-Bench-ZH' = 0.919. Own-model lab result (competitor rows not used); same report and ledger entry as the existing GenEval/DPG rows.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "llada-image-turbo",
    "bench": "cvtg-2k-word-accuracy",
    "score": 0.84,
    "source": "https://arxiv.org/html/2609.03796v1#S5.T6",
    "by": "lab",
    "date": "2026-09-03",
    "note": "LLaDA-Image report (Inclusion AI, arXiv 2609.03796v1, CC BY-SA 4.0), Table 6 (CVTG-2K), row 'LLaDA-Image Turbo', Word Accuracy 'average' column = 0.84 (2/3/4/5 regions and average: 0.846 | 0.843 | 0.835 | 0.840 | 0.840); NED and CLIPScore not used. Own-model lab result.",
    "source_id": "visual-llada-image"
  },
  {
    "model": "janus-1-3b",
    "bench": "geneval",
    "score": 0.61,
    "source": "https://arxiv.org/html/2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "note": "DeepSeek-AI's own model (Janus 1.3B, cited as Janus [46]) in the Janus-Pro report's Table 4 GenEval, 'Overall' = 0.61; same table and ledger entry as the existing Janus-Pro-1B/7B rows. Equals the Janus paper's own figure (recorded separately from arXiv 2410.13848v1 as a single-model run).",
    "source_id": "report-2501.17811v1"
  },
  {
    "model": "janus-1-3b",
    "bench": "dpg-bench",
    "score": 0.7968,
    "source": "https://arxiv.org/html/2501.17811v1",
    "by": "lab",
    "date": "2025-01-29",
    "note": "DeepSeek-AI's own model (Janus 1.3B) in the Janus-Pro report's Table 5 DPG-Bench, 'Overall' = 79.68, divided by 100; same table and ledger entry as the existing Janus-Pro-1B/7B rows.",
    "source_id": "report-2501.17811v1"
  },
  {
    "model": "data2vec-audio-base-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0277,
    "source": "https://huggingface.co/facebook/data2vec-audio-base-960h/blob/32331f3123e703528918aa688a9a38232d58c872/README.md#evaluation",
    "by": "lab",
    "date": "2022-05-24",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @32331f3, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 2.77% for facebook/data2vec-audio-base-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "data2vec-audio-base-960h",
    "bench": "librispeech-test-other-wer",
    "score": 0.0708,
    "source": "https://huggingface.co/facebook/data2vec-audio-base-960h/blob/32331f3123e703528918aa688a9a38232d58c872/README.md#evaluation",
    "by": "lab",
    "date": "2022-05-24",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @32331f3, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 7.08% for facebook/data2vec-audio-base-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "data2vec-audio-large-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0189,
    "source": "https://huggingface.co/facebook/data2vec-audio-large-960h/blob/27aba26eed532b86dcd0f17284a0307de4b51f39/README.md#evaluation",
    "by": "lab",
    "date": "2022-06-06",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @27aba26, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 1.89% for facebook/data2vec-audio-large-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "data2vec-audio-large-960h",
    "bench": "librispeech-test-other-wer",
    "score": 0.0407,
    "source": "https://huggingface.co/facebook/data2vec-audio-large-960h/blob/27aba26eed532b86dcd0f17284a0307de4b51f39/README.md#evaluation",
    "by": "lab",
    "date": "2022-06-06",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @27aba26, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 4.07% for facebook/data2vec-audio-large-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-conformer-rope-large-960h-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0196,
    "source": "https://huggingface.co/facebook/wav2vec2-conformer-rope-large-960h-ft/blob/6b36ef01c6443c67ae7ed0822876d091ab50e4aa/README.md#evaluation",
    "by": "lab",
    "date": "2023-03-21",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @6b36ef0, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 1.96% for facebook/wav2vec2-conformer-rope-large-960h-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-conformer-rope-large-960h-ft",
    "bench": "librispeech-test-other-wer",
    "score": 0.0398,
    "source": "https://huggingface.co/facebook/wav2vec2-conformer-rope-large-960h-ft/blob/6b36ef01c6443c67ae7ed0822876d091ab50e4aa/README.md#evaluation",
    "by": "lab",
    "date": "2023-03-21",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @6b36ef0, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 3.98% for facebook/wav2vec2-conformer-rope-large-960h-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-conformer-rel-pos-large-960h-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0185,
    "source": "https://huggingface.co/facebook/wav2vec2-conformer-rel-pos-large-960h-ft/blob/ca7f36f527f234b3cd4f05ecee30361f971e8e33/README.md#evaluation",
    "by": "lab",
    "date": "2022-06-15",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @ca7f36f, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 1.85% for facebook/wav2vec2-conformer-rel-pos-large-960h-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. Test-other: the evaluation section prints 3.82 while the model-index metadata says 3.83; the printed evaluation result is recorded.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-conformer-rel-pos-large-960h-ft",
    "bench": "librispeech-test-other-wer",
    "score": 0.0382,
    "source": "https://huggingface.co/facebook/wav2vec2-conformer-rel-pos-large-960h-ft/blob/ca7f36f527f234b3cd4f05ecee30361f971e8e33/README.md#evaluation",
    "by": "lab",
    "date": "2022-06-15",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @ca7f36f, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 3.82% for facebook/wav2vec2-conformer-rel-pos-large-960h-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. Test-other: the evaluation section prints 3.82 while the model-index metadata says 3.83; the printed evaluation result is recorded.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-base-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.034,
    "source": "https://huggingface.co/facebook/wav2vec2-base-960h/blob/22aad52d435eb6dbaf354bdad9b0da84ce7d6156/README.md#evaluation",
    "by": "lab",
    "date": "2022-11-14",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @22aad52, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 3.4% for facebook/wav2vec2-base-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-base-960h",
    "bench": "librispeech-test-other-wer",
    "score": 0.086,
    "source": "https://huggingface.co/facebook/wav2vec2-base-960h/blob/22aad52d435eb6dbaf354bdad9b0da84ce7d6156/README.md#evaluation",
    "by": "lab",
    "date": "2022-11-14",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @22aad52, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 8.6% for facebook/wav2vec2-base-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "librispeech-test-clean-wer",
    "score": 0.028,
    "source": "https://huggingface.co/facebook/wav2vec2-large-960h/blob/bdeaacdf88f7a155f50a2704bc967aa81fbbb2ab/README.md#evaluation",
    "by": "lab",
    "date": "2022-04-05",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @bdeaacd, evaluation section '*Result (WER)*' table: LibriSpeech test-clean WER = 2.8% for facebook/wav2vec2-large-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. No model-index entry; values from the evaluation section only (the abstract quotes the paper's 1.8/3.3 with LM, not this checkpoint's greedy result).",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-large-960h",
    "bench": "librispeech-test-other-wer",
    "score": 0.063,
    "source": "https://huggingface.co/facebook/wav2vec2-large-960h/blob/bdeaacdf88f7a155f50a2704bc967aa81fbbb2ab/README.md#evaluation",
    "by": "lab",
    "date": "2022-04-05",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @bdeaacd, evaluation section '*Result (WER)*' table: LibriSpeech test-other WER = 6.3% for facebook/wav2vec2-large-960h. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. No model-index entry; values from the evaluation section only (the abstract quotes the paper's 1.8/3.3 with LM, not this checkpoint's greedy result).",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "librispeech-test-clean-wer",
    "score": 0.019,
    "source": "https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self/blob/54074b1c16f4de6a5ad59affb4caa8f2ea03a119/README.md#evaluation",
    "by": "lab",
    "date": "2022-05-23",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @54074b1, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-clean WER = 1.9% for facebook/wav2vec2-large-960h-lv60-self. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "wav2vec2-large-960h-lv60-self",
    "bench": "librispeech-test-other-wer",
    "score": 0.039,
    "source": "https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self/blob/54074b1c16f4de6a5ad59affb4caa8f2ea03a119/README.md#evaluation",
    "by": "lab",
    "date": "2022-05-23",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @54074b1, evaluation section '*Result (WER)*' table (clean/other) and the matching model-index 'Test WER' entries: LibriSpeech test-other WER = 3.9% for facebook/wav2vec2-large-960h-lv60-self. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date.",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "hubert-large-ls960-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.019,
    "source": "https://huggingface.co/facebook/hubert-large-ls960-ft/blob/ece5fabbf034c1073acae96d5401b25be96709d8/README.md#evaluation",
    "by": "lab",
    "date": "2022-05-24",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @ece5fab, model-index 'Test WER' entry (LibriSpeech clean, split test): LibriSpeech test-clean WER = 1.9% for facebook/hubert-large-ls960-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. Card gives test-clean only (model-index).",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "hubert-xlarge-ls960-ft",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018,
    "source": "https://huggingface.co/facebook/hubert-xlarge-ls960-ft/blob/86a09e67e0c8d074533992379242405825516eca/README.md#evaluation",
    "by": "lab",
    "date": "2023-06-27",
    "note": "Model card in Meta's facebook organisation (Apache-2.0) @86a09e6, model-index 'Test WER' entry (LibriSpeech clean, split test): LibriSpeech test-clean WER = 1.8% for facebook/hubert-xlarge-ls960-ft. Greedy CTC decoding with the card's own transformers evaluation snippet, no LM. Card last revision date used as the date. Card gives test-clean only (model-index).",
    "source_id": "first-party-meta-asr-cards"
  },
  {
    "model": "t2v-turbo-v2",
    "bench": "vbench-t2v-total",
    "score": 0.8352,
    "source": "https://arxiv.org/html/2410.05677v3#S4.T1",
    "by": "lab",
    "date": "2025-09-16",
    "note": "T2V-Turbo-v2 paper (Li et al., UCSB, arXiv 2410.05677v3, CC BY 4.0), Table 1 (VBench, 'Total Score' row), column 'T2V-Turbo-v2 w/ MG' = 83.52 (16 inference steps). The released jiachenli-ucsb/T2V-Turbo-v2 checkpoint is unet_mg.pt (with motion guidance). Authors' own model.",
    "source_id": "paper-2410.05677v3-t2vturbo2"
  },
  {
    "model": "t2v-turbo-vc2",
    "bench": "vbench-t2v-total",
    "score": 0.8051,
    "source": "https://arxiv.org/html/2410.05677v3#S4.T1",
    "by": "lab",
    "date": "2025-09-16",
    "note": "T2V-Turbo-v2 paper (Li et al., UCSB, arXiv 2410.05677v3, CC BY 4.0), Table 1 (VBench, 'Total Score' row), column 'T2V-Turbo 16-step' = 80.51 (the same table gives 81.01 at 4 steps; 16 steps recorded to match the T2V-Turbo-v2 column). T2V-Turbo (VC2), the authors' own earlier model. Competitor columns (Pika, Gen-2, Gen-3, Kling, VideoCrafter2) are VBench-leaderboard copies and not used.",
    "source_id": "paper-2410.05677v3-t2vturbo2"
  },
  {
    "model": "wizardcoder-python-13b-v1-0",
    "bench": "humaneval",
    "score": 0.64,
    "source": "https://huggingface.co/WizardLMTeam/WizardCoder-Python-13B-V1.0/blob/5ac6748b1f5a4c282107ddc7d3b69fdc4a686d75/README.md#wizardcoder-empowering-code-large-language-models-with-evol-instruct",
    "by": "lab",
    "date": "2024-01-19",
    "note": "Vendor-reported (WizardLM team, own model), hand-recorded from the pinned WizardCoder-Python-13B-V1.0 card (org WizardLMTeam, formerly WizardLM), family table row 'WizardCoder-Python-13B-V1.0', column 'HumanEval' = 64.0 (pass@1). Only the HumanEval column is used: the HumanEval+/MBPP/MBPP+ columns may repeat EvalPlus-leaderboard measurements. Competitor rows (GPT-4, Gemini Pro, DeepSeek-Coder) not used. Matches the model-index pass@1 0.64 in the card front matter.",
    "source_id": "first-party-wizardlm-scores"
  },
  {
    "model": "wizardcoder-python-34b-v1-0",
    "bench": "humaneval",
    "score": 0.732,
    "source": "https://huggingface.co/WizardLMTeam/WizardCoder-Python-13B-V1.0/blob/5ac6748b1f5a4c282107ddc7d3b69fdc4a686d75/README.md#wizardcoder-empowering-code-large-language-models-with-evol-instruct",
    "by": "lab",
    "date": "2024-01-19",
    "note": "Vendor-reported (WizardLM team, own model), hand-recorded from the pinned WizardCoder-Python-13B-V1.0 card (org WizardLMTeam, formerly WizardLM), family table row 'WizardCoder-Python-34B-V1.0', column 'HumanEval' = 73.2 (pass@1). Only the HumanEval column is used: the HumanEval+/MBPP/MBPP+ columns may repeat EvalPlus-leaderboard measurements. Competitor rows (GPT-4, Gemini Pro, DeepSeek-Coder) not used.",
    "source_id": "first-party-wizardlm-scores"
  },
  {
    "model": "wizardcoder-33b-v1-1",
    "bench": "humaneval",
    "score": 0.799,
    "source": "https://huggingface.co/WizardLMTeam/WizardCoder-Python-13B-V1.0/blob/5ac6748b1f5a4c282107ddc7d3b69fdc4a686d75/README.md#wizardcoder-empowering-code-large-language-models-with-evol-instruct",
    "by": "lab",
    "date": "2024-01-19",
    "note": "Vendor-reported (WizardLM team, own model), hand-recorded from the pinned WizardCoder-Python-13B-V1.0 card (org WizardLMTeam, formerly WizardLM), family table row 'WizardCoder-33B-V1.1', column 'HumanEval' = 79.9 (pass@1). Only the HumanEval column is used: the HumanEval+/MBPP/MBPP+ columns may repeat EvalPlus-leaderboard measurements. Competitor rows (GPT-4, Gemini Pro, DeepSeek-Coder) not used.",
    "source_id": "first-party-wizardlm-scores"
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humanevalexplain",
    "score": 0.119,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoderBase-3B' (Completion prompt), column 'Avg.' = 11.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.0, 10.4, 14.6, 11.0, 13.4, 11.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "stable-code-3b",
    "bench": "humanevalexplain",
    "score": 0.123,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StableCode-3B' (Completion prompt), column 'Avg.' = 12.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.0, 7.9, 22.0, 4.3, 14.6, 14.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-3b",
    "bench": "humanevalexplain",
    "score": 0.131,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoder2-3B' (Completion prompt), column 'Avg.' = 13.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 12.2, 13.4, 19.5, 6.7, 14.0, 12.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-2b",
    "bench": "humanevalexplain",
    "score": 0.173,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeGemma-2B' (Completion prompt), column 'Avg.' = 17.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 20.7, 15.9, 20.7, 12.8, 17.7, 15.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "humanevalexplain",
    "score": 0.219,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-3B-Code-Base' (Completion prompt), column 'Avg.' = 21.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 25.0, 18.9, 29.9, 17.1, 26.8, 14.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humanevalexplain",
    "score": 0.148,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoderBase-7B' (Completion prompt), column 'Avg.' = 14.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 14.0, 17.1, 17.7, 10.4, 17.1, 12.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-7b",
    "bench": "humanevalexplain",
    "score": 0.139,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-7B' (Completion prompt), column 'Avg.' = 13.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.0, 14.0, 16.5, 9.8, 17.7, 14.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-7b",
    "bench": "humanevalexplain",
    "score": 0.135,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoder2-7B' (Completion prompt), column 'Avg.' = 13.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 4.9, 12.8, 22.0, 4.9, 22.0, 14.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-7b",
    "bench": "humanevalexplain",
    "score": 0.124,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeGemma-7B' (Completion prompt), column 'Avg.' = 12.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 13.1, 13.8, 2.0, 8.0, 18.6, 18.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "humanevalexplain",
    "score": 0.264,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-8B-Code-Base' (Completion prompt), column 'Avg.' = 26.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 23.5, 32.3, 25.0, 23.2, 28.0, 19.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoderbase",
    "bench": "humanevalexplain",
    "score": 0.153,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoderBase-15B' (Completion prompt), column 'Avg.' = 15.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 9.8, 15.2, 24.4, 9.1, 20.1, 13.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-13b",
    "bench": "humanevalexplain",
    "score": 0.15,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-13B' (Completion prompt), column 'Avg.' = 15.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 13.4, 14.0, 23.2, 9.8, 15.9, 13.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-15b",
    "bench": "humanevalexplain",
    "score": 0.17,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'StarCoder2-15B' (Completion prompt), column 'Avg.' = 17.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 20.1, 19.5, 7.3, 9.8, 23.8, 21.3). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "humanevalexplain",
    "score": 0.188,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-20B-Code-Base' (Completion prompt), column 'Avg.' = 18.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 17.1, 18.3, 23.2, 10.4, 25.6, 18.3). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-34b",
    "bench": "humanevalexplain",
    "score": 0.171,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-34B' (Completion prompt), column 'Avg.' = 17.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.6, 18.3, 22.0, 9.8, 20.1, 20.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "humanevalexplain",
    "score": 0.341,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-34B-Code-Base' (Completion prompt), column 'Avg.' = 34.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 42.7, 26.2, 47.0, 26.8, 36.6, 25.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-70b",
    "bench": "humanevalexplain",
    "score": 0.282,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-70B' (Completion prompt), column 'Avg.' = 28.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 24.4, 30.5, 43.9, 19.5, 31.1, 20.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-2b-base",
    "bench": "humanevalexplain",
    "score": 0.109,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Gemma-2B' (Completion prompt), column 'Avg.' = 10.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 9.8, 9.8, 14.6, 7.9, 14.0, 9.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-7b-base",
    "bench": "humanevalexplain",
    "score": 0.15,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Gemma-7B' (Completion prompt), column 'Avg.' = 15.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 10.4, 18.3, 19.5, 9.8, 18.3, 14.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "humanevalexplain",
    "score": 0.207,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Mixtral-8x7B-v0.1' (Completion prompt), column 'Avg.' = 20.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 17.1, 18.3, 35.4, 19.5, 18.9, 15.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humanevalexplain",
    "score": 0.384,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeGemma-7B-IT' (Instruction prompt), column 'Avg.' = 38.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 48.2, 40.9, 51.8, 31.1, 33.5, 25.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "humanevalexplain",
    "score": 0.251,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-7B-Instruct' (Instruction prompt), column 'Avg.' = 25.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 29.9, 29.9, 32.9, 19.5, 25.0, 13.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "humanevalexplain",
    "score": 0.277,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-13B-Instruct' (Instruction prompt), column 'Avg.' = 27.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 38.4, 28.0, 36.0, 22.6, 26.8, 14.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "humanevalexplain",
    "score": 0.303,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-34B-Instruct' (Instruction prompt), column 'Avg.' = 30.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 42.1, 31.1, 39.6, 20.1, 31.7, 17.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humanevalexplain",
    "score": 0.411,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'CodeLlama-70B-Instruct' (Instruction prompt), column 'Avg.' = 41.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 47.0, 40.2, 54.9, 34.1, 46.3, 23.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "humanevalexplain",
    "score": 0.26,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-3b-Code-Instruct' (Instruction prompt), column 'Avg.' = 26.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 39.6, 26.8, 39.0, 14.0, 23.8, 12.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-8b-code-instruct-4k",
    "bench": "humanevalexplain",
    "score": 0.409,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-8b-Code-Instruct' (Instruction prompt), column 'Avg.' = 40.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 53.0, 42.7, 52.4, 36.6, 43.9, 16.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "humanevalexplain",
    "score": 0.382,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-20B-Code-Instruct' (Instruction prompt), column 'Avg.' = 38.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 44.5, 42.7, 49.4, 32.3, 42.1, 18.3). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "humanevalexplain",
    "score": 0.419,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Granite-34B-Code-Instruct' (Instruction prompt), column 'Avg.' = 41.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 53.0, 45.1, 50.6, 36.0, 42.7, 23.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-2b",
    "bench": "humanevalexplain",
    "score": 0.102,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Gemma-2B-IT' (Instruction prompt), column 'Avg.' = 10.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 9.8, 12.8, 13.4, 8.5, 13.4, 3.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-7b",
    "bench": "humanevalexplain",
    "score": 0.215,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Gemma-7B-IT' (Instruction prompt), column 'Avg.' = 21.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 31.1, 23.2, 28.0, 14.6, 23.8, 8.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humanevalexplain",
    "score": 0.241,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Mistral-7B-Instruct-v0.2' (Instruction prompt), column 'Avg.' = 24.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 24.4, 26.8, 31.1, 22.6, 25.6, 14.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humanevalexplain",
    "score": 0.37,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Mixtral-8x7B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 37.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 47.0, 40.9, 48.2, 28.0, 32.9, 25.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "humanevalexplain",
    "score": 0.566,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Mixtral-8x22B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 56.6 (mean of Python/JavaScript/Java/Go/C++/Rust = 67.1, 56.7, 67.7, 44.5, 64.0, 39.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "llama-3-8b",
    "bench": "humanevalexplain",
    "score": 0.291,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Llama-3-8B-Instruct' (Instruction prompt), column 'Avg.' = 29.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 36.0, 31.7, 40.2, 19.5, 31.1, 15.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "llama-3-70b",
    "bench": "humanevalexplain",
    "score": 0.454,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T10",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 10 'Pass@1 performance on HumanEvalExplain', row 'Llama-3-70B-Instruct' (Instruction prompt), column 'Avg.' = 45.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 50.6, 47.6, 57.9, 34.8, 48.2, 33.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoderbase-3b",
    "bench": "humanevalfix",
    "score": 0.064,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoderBase-3B' (Completion prompt), column 'Avg.' = 6.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 12.2, 9.8, 6.1, 7.9, 1.8, 0.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "stable-code-3b",
    "bench": "humanevalfix",
    "score": 0.105,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StableCode-3B' (Completion prompt), column 'Avg.' = 10.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.0, 7.3, 20.1, 10.4, 12.8, 1.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-3b",
    "bench": "humanevalfix",
    "score": 0.11,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoder2-3B' (Completion prompt), column 'Avg.' = 11.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 18.3, 15.9, 12.8, 12.8, 5.5, 0.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-2b",
    "bench": "humanevalfix",
    "score": 0.044,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeGemma-2B' (Completion prompt), column 'Avg.' = 4.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 4.3, 7.3, 3.0, 9.8, 1.8, 0.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "humanevalfix",
    "score": 0.193,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-3B-Code-Base' (Completion prompt), column 'Avg.' = 19.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 18.3, 23.2, 29.9, 24.4, 16.5, 3.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoderbase-7b",
    "bench": "humanevalfix",
    "score": 0.125,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoderBase-7B' (Completion prompt), column 'Avg.' = 12.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 15.9, 21.3, 17.1, 14.6, 5.5, 0.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-7b",
    "bench": "humanevalfix",
    "score": 0.142,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-7B' (Completion prompt), column 'Avg.' = 14.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 15.9, 14.0, 23.8, 15.2, 5.5, 11.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-7b",
    "bench": "humanevalfix",
    "score": 0.089,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoder2-7B' (Completion prompt), column 'Avg.' = 8.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 5.5, 13.4, 15.9, 11.0, 7.3, 0.6). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-7b",
    "bench": "humanevalfix",
    "score": 0.101,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeGemma-7B' (Completion prompt), column 'Avg.' = 10.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 8.5, 5.5, 20.1, 14.6, 7.9, 3.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "humanevalfix",
    "score": 0.296,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-8B-Code-Base' (Completion prompt), column 'Avg.' = 29.6 (mean of Python/JavaScript/Java/Go/C++/Rust = 22.6, 35.4, 38.4, 37.2, 28.7, 15.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoderbase",
    "bench": "humanevalfix",
    "score": 0.129,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoderBase-15B' (Completion prompt), column 'Avg.' = 12.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 10.4, 17.7, 17.1, 18.9, 9.8, 3.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-13b",
    "bench": "humanevalfix",
    "score": 0.095,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-13B' (Completion prompt), column 'Avg.' = 9.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 6.1, 9.1, 17.1, 9.8, 6.1, 10.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "starcoder2-15b",
    "bench": "humanevalfix",
    "score": 0.219,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'StarCoder2-15B' (Completion prompt), column 'Avg.' = 21.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 9.1, 18.9, 25.0, 37.2, 25.0, 16.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "humanevalfix",
    "score": 0.177,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-20B-Code-Base' (Completion prompt), column 'Avg.' = 17.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 23.2, 23.8, 14.6, 26.2, 15.2, 3.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-34b",
    "bench": "humanevalfix",
    "score": 0.197,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-34B' (Completion prompt), column 'Avg.' = 19.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 14.0, 20.7, 20.1, 26.2, 31.1, 6.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "humanevalfix",
    "score": 0.295,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-34B-Code-Base' (Completion prompt), column 'Avg.' = 29.5 (mean of Python/JavaScript/Java/Go/C++/Rust = 20.1, 30.5, 40.9, 34.1, 39.0, 12.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-70b",
    "bench": "humanevalfix",
    "score": 0.328,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-70B' (Completion prompt), column 'Avg.' = 32.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 12.8, 31.1, 41.5, 42.1, 38.4, 31.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-2b-base",
    "bench": "humanevalfix",
    "score": 0.019,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Gemma-2B' (Completion prompt), column 'Avg.' = 1.9 (mean of Python/JavaScript/Java/Go/C++/Rust = 1.2, 2.4, 4.3, 2.4, 1.2, 0.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-7b-base",
    "bench": "humanevalfix",
    "score": 0.074,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Gemma-7B' (Completion prompt), column 'Avg.' = 7.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 1.8, 1.2, 26.2, 7.3, 6.7, 1.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "humanevalfix",
    "score": 0.143,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Mixtral-8x7B-v0.1' (Completion prompt), column 'Avg.' = 14.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 11.0, 12.8, 18.3, 25.0, 15.9, 3.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "humanevalfix",
    "score": 0.458,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeGemma-7B-IT' (Instruction prompt), column 'Avg.' = 45.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 46.3, 45.7, 52.4, 48.2, 43.9, 38.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "humanevalfix",
    "score": 0.128,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-7B-Instruct' (Instruction prompt), column 'Avg.' = 12.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 19.5, 18.9, 13.4, 14.6, 6.1, 4.3). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "humanevalfix",
    "score": 0.158,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-13B-Instruct' (Instruction prompt), column 'Avg.' = 15.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 18.9, 18.9, 24.4, 22.6, 9.8, 0.0). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "humanevalfix",
    "score": 0.282,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-34B-Instruct' (Instruction prompt), column 'Avg.' = 28.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 37.8, 28.0, 37.2, 24.4, 24.4, 17.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "humanevalfix",
    "score": 0.53,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'CodeLlama-70B-Instruct' (Instruction prompt), column 'Avg.' = 53.0 (mean of Python/JavaScript/Java/Go/C++/Rust = 64.6, 52.4, 57.3, 51.8, 51.2, 40.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "humanevalfix",
    "score": 0.273,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-3b-Code-Instruct' (Instruction prompt), column 'Avg.' = 27.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 26.8, 28.0, 33.5, 27.4, 31.7, 16.5). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-8b-code-instruct-4k",
    "bench": "humanevalfix",
    "score": 0.404,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-8b-Code-Instruct' (Instruction prompt), column 'Avg.' = 40.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 39.6, 40.9, 48.2, 41.5, 39.0, 32.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "humanevalfix",
    "score": 0.411,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-20B-Code-Instruct' (Instruction prompt), column 'Avg.' = 41.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 43.9, 43.9, 45.7, 41.5, 41.5, 29.9). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "humanevalfix",
    "score": 0.502,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "lab",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Granite-34B-Code-Instruct' (Instruction prompt), column 'Avg.' = 50.2 (mean of Python/JavaScript/Java/Go/C++/Rust = 54.9, 47.6, 55.5, 51.2, 47.0, 45.1). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-2b",
    "bench": "humanevalfix",
    "score": 0.167,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Gemma-2B-IT' (Instruction prompt), column 'Avg.' = 16.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 18.9, 15.9, 25.6, 13.4, 15.9, 10.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "gemma-7b",
    "bench": "humanevalfix",
    "score": 0.137,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Gemma-7B-IT' (Instruction prompt), column 'Avg.' = 13.7 (mean of Python/JavaScript/Java/Go/C++/Rust = 10.3, 9.8, 22.0, 14.3, 16.2, 9.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "humanevalfix",
    "score": 0.268,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Mistral-7B-Instruct-v0.2' (Instruction prompt), column 'Avg.' = 26.8 (mean of Python/JavaScript/Java/Go/C++/Rust = 33.5, 22.6, 27.4, 27.4, 26.2, 23.8). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "humanevalfix",
    "score": 0.393,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Mixtral-8x7B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 39.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 44.5, 37.8, 47.6, 38.4, 39.0, 28.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "humanevalfix",
    "score": 0.561,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Mixtral-8x22B-Instruct-v0.1' (Instruction prompt), column 'Avg.' = 56.1 (mean of Python/JavaScript/Java/Go/C++/Rust = 59.1, 53.7, 66.5, 55.5, 56.1, 45.7). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "llama-3-8b",
    "bench": "humanevalfix",
    "score": 0.293,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Llama-3-8B-Instruct' (Instruction prompt), column 'Avg.' = 29.3 (mean of Python/JavaScript/Java/Go/C++/Rust = 40.2, 25.6, 43.3, 29.9, 13.4, 23.2). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "llama-3-70b",
    "bench": "humanevalfix",
    "score": 0.524,
    "source": "https://arxiv.org/html/2405.04324v1#S6.T11",
    "by": "independent",
    "date": "2024-05-07",
    "note": "Granite Code paper (arXiv:2405.04324v1, IBM) Table 11 'Pass@1 performance on HumanEvalFix', row 'Llama-3-70B-Instruct' (Instruction prompt), column 'Avg.' = 52.4 (mean of Python/JavaScript/Java/Go/C++/Rust = 57.3, 51.2, 54.3, 51.8, 50.6, 49.4). Same IBM harness, prompts and model mapping as the existing Table 3 HumanEvalSynthesize rows.",
    "source_id": "paper-2405.04324v1"
  },
  {
    "model": "intellect-3",
    "bench": "math-500",
    "score": 0.981,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'MATH-500' = 98.1 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "aime-2024",
    "score": 0.908,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'AIME24' = 90.8 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "aime-2025",
    "score": 0.88,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'AIME25' = 88.0 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.693,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'LCB' = 69.3 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "gpqa-diamond",
    "score": 0.744,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'GPQA' = 74.4 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "hle",
    "score": 0.146,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'HLE' = 14.6 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "intellect-3",
    "bench": "mmlu-pro",
    "score": 0.819,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "lab",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'INTELLECT-3', column 'MMLU-Pro' = 81.9 (Avg@1 over 12K). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Prime Intellect's own model.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "math-500",
    "score": 0.978,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'MATH-500' = 97.8 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "aime-2024",
    "score": 0.846,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'AIME24' = 84.6 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "aime-2025",
    "score": 0.82,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'AIME25' = 82.0 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.615,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'LCB' = 61.5 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "gpqa-diamond",
    "score": 0.733,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'GPQA' = 73.3 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "hle",
    "score": 0.133,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'HLE' = 13.3 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5-air",
    "bench": "mmlu-pro",
    "score": 0.739,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5-Air', column 'MMLU-Pro' = 73.9 (Avg@1 over 12K). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API, temperature 0.6 (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "math-500",
    "score": 0.97,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'MATH-500' = 97.0 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2024",
    "score": 0.858,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'AIME24' = 85.8 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2025",
    "score": 0.833,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'AIME25' = 83.3 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.645,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'LCB' = 64.5 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "gpqa-diamond",
    "score": 0.77,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'GPQA' = 77.0 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "glm-4-5",
    "bench": "hle",
    "score": 0.148,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GLM-4.5', column 'HLE' = 14.8 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official z-AI API (report A.2). MMLU-Pro 83.5 not recorded: the report marks it '* Reported by AA Index'.",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "math-500",
    "score": 0.873,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'MATH-500' = 87.3 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2024",
    "score": 0.832,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'AIME24' = 83.2 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2025",
    "score": 0.734,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'AIME25' = 73.4 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.625,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'LCB' = 62.5 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "gpqa-diamond",
    "score": 0.775,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'GPQA' = 77.5 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hle",
    "score": 0.159,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'HLE' = 15.9 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "mmlu-pro",
    "score": 0.753,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek R1 0528', column 'MMLU-Pro' = 75.3 (Avg@1 over 12K). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Evaluated via OpenRouter routed to the official DeepSeek API (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "math-500",
    "score": 0.968,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'MATH-500' = 96.8 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "aime-2024",
    "score": 0.881,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'AIME24' = 88.1 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "aime-2025",
    "score": 0.847,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'AIME25' = 84.7 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.716,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'LCB' = 71.6 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "gpqa-diamond",
    "score": 0.814,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'GPQA' = 81.4 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "hle",
    "score": 0.179,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'HLE' = 17.9 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "mmlu-pro",
    "score": 0.846,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'DeepSeek v3.2', column 'MMLU-Pro' = 84.6 (Avg@1 over 12K). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Report A.2: 'DeepSeek v3.2 (Thinking)' via the official DeepSeek API, model slug deepseek-reasoner; the card (27 Nov 2025) and report PDF (30 Nov 2025) predate the 1 Dec 2025 DeepSeek-V3.2 release, so the API then served DeepSeek-V3.2-Exp (thinking mode).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "math-500",
    "score": 0.96,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'MATH-500' = 96.0 (2 generations per problem). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aime-2024",
    "score": 0.758,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'AIME24' = 75.8 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "aime-2025",
    "score": 0.777,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'AIME25' = 77.7 (Avg@32). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.699,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'LCB' = 69.9 (v6, 454 problems Aug 2024-May 2025, Avg@2). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "gpqa-diamond",
    "score": 0.7,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'GPQA' = 70.0 (Diamond, Avg@4). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "hle",
    "score": 0.106,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'HLE' = 10.6 (text-only subset (2,158), no tools). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "mmlu-pro",
    "score": 0.671,
    "source": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/ff39d4a4688989f3f28868923d030c28e1b7d81c/README.md#evaluation",
    "by": "independent",
    "date": "2025-11-27",
    "note": "INTELLECT-3 model card (Prime Intellect, MIT) @ff39d4a, 'Evaluation' table, row 'GPT-O5S 120B', column 'MMLU-Pro' = 67.1 (Avg@1 over 12K). Prime Intellect ran every row in its own Environments Hub harness (technical report: 'All the models above were evaluated using our public, reproducible Environments Hub implementations'; 'running all models in the same evaluation harnesses'). Row label printed 'GPT-O5S 120B' (report Table 2: 'GPT-OSS 120B'); evaluated at high reasoning effort via TogetherAI (report A.2).",
    "source_id": "primeintellect-intellect3-card-eval"
  },
  {
    "model": "olmo-7b",
    "bench": "mmlu",
    "score": 0.283,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row 'OLMo-7B', column 'MMLU 0-shot' = 28.3. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. Base model (allenai/OLMo-7B) scored with the Tülu suite.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b",
    "bench": "gsm8k",
    "score": 0.085,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row 'OLMo-7B', column 'GSM8k 8-shot CoT' = 8.5. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. Base model (allenai/OLMo-7B) scored with the Tülu suite.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b",
    "bench": "bbh",
    "score": 0.317,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row 'OLMo-7B', column 'BBH 3-shot CoT' = 31.7. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. Base model (allenai/OLMo-7B) scored with the Tülu suite.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.323,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row 'OLMo-7B', column 'TydiQA 1-shot (F1)' = 32.3. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. Base model (allenai/OLMo-7B) scored with the Tülu suite.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.214,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row 'OLMo-7B', column 'Codex-Eval Pass@10' = 21.4. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. Base model (allenai/OLMo-7B) scored with the Tülu suite.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "mmlu",
    "score": 0.461,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row '+SFT+DPO', column 'MMLU 0-shot' = 46.1. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. +SFT+DPO is the released OLMo 7B Instruct (Tulu 2 SFT mix + cleaned UltraFeedback DPO, per its model card). Table 4 prints MMLU 46.2 for the same model; the Table 8 value is recorded.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "gsm8k",
    "score": 0.11,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row '+SFT+DPO', column 'GSM8k 8-shot CoT' = 11.0. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. +SFT+DPO is the released OLMo 7B Instruct (Tulu 2 SFT mix + cleaned UltraFeedback DPO, per its model card). Table 4 prints MMLU 46.2 for the same model; the Table 8 value is recorded.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "bbh",
    "score": 0.358,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row '+SFT+DPO', column 'BBH 3-shot CoT' = 35.8. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. +SFT+DPO is the released OLMo 7B Instruct (Tulu 2 SFT mix + cleaned UltraFeedback DPO, per its model card). Table 4 prints MMLU 46.2 for the same model; the Table 8 value is recorded.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "tulu2-tydiqa-goldp-1shot-f1",
    "score": 0.217,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row '+SFT+DPO', column 'TydiQA 1-shot (F1)' = 21.7. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. +SFT+DPO is the released OLMo 7B Instruct (Tulu 2 SFT mix + cleaned UltraFeedback DPO, per its model card). Table 4 prints MMLU 46.2 for the same model; the Table 8 value is recorded.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "tulu2-codexeval-pass10",
    "score": 0.278,
    "source": "https://arxiv.org/html/2402.00838v4#A5.T8",
    "by": "lab",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 8 ('Evaluation of OLMo-7B models before and after instruction finetuning and DPO training on the full Tülu evaluation suite'), row '+SFT+DPO', column 'Codex-Eval Pass@10' = 27.8. Same Tülu evaluation suite and settings as the Tulu 2 paper rows already imported for these bench ids; Ai2's own model. +SFT+DPO is the released OLMo 7B Instruct (Tulu 2 SFT mix + cleaned UltraFeedback DPO, per its model card). Table 4 prints MMLU 46.2 for the same model; the Table 8 value is recorded.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "mpt-7b-chat",
    "bench": "mmlu",
    "score": 0.338,
    "source": "https://arxiv.org/html/2402.00838v4#S4.T4",
    "by": "independent",
    "date": "2024-06-07",
    "note": "OLMo paper (Ai2, arXiv 2402.00838v4, CC BY 4.0), Table 4 ('Evaluation of various instruction-tuned 7B models'), row 'MPT Chat' (mosaicml/mpt-7b-chat per Appendix E), column 'MMLU 0-shot' = 33.8, run by Ai2 in the Tülu suite (Sec. 4.3: 'We additionally compare to officially released instruction-tuned variants of the models from Table 3'). Llama-2-Chat and Tülu 2 rows of Table 4 repeat the Tulu 2 paper's values (already imported) and are not duplicated.",
    "source_id": "paper-2402.00838v4"
  },
  {
    "model": "kokoro-82m",
    "bench": "paradee-wikitext200-utmos22",
    "score": 0.88,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'Kokoro-82M, teacher (af_heart)', column UTMOS = 4.52 ± 0.003 (utmos22_strong checkpoint, mean over the 200 sentences, 95% CI). Stored (UTMOS-1)/4. Teacher model; reference [11] is 'hexgrad, Kokoro-82M, Hugging Face model card, v1.0' (catalogue kokoro-82m, voice af_heart). Paradee's author is not Kokoro's maker, so by=independent.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "piper-en-us-lessac-medium",
    "bench": "paradee-wikitext200-utmos22",
    "score": 0.84,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'Piper, en_US-lessac-medium', column UTMOS = 4.36 ± 0.02 (utmos22_strong checkpoint, mean over the 200 sentences, 95% CI). Stored (UTMOS-1)/4. Voice en_US-lessac-medium (rhasspy/piper-voices; catalogue piper-en-us-lessac-medium); 15.7M params, 63.2 MB fp32 as printed; timed from text.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "kitten-tts-nano-0-8",
    "bench": "paradee-wikitext200-utmos22",
    "score": 0.7525,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'KittenTTS nano 0.8 (Bella)', column UTMOS = 4.01 ± 0.04 (utmos22_strong checkpoint, mean over the 200 sentences, 95% CI). Stored (UTMOS-1)/4. Proposed comparison-only model kitten-tts-nano-0-8: printed '14.0M' params and '56.8 MB' fp32 file match KittenML/kitten-tts-nano-0.8-fp32 kitten_tts_nano_v0_8.onnx (56,767,095 bytes @7a1db64); voice Bella. Not the catalogue's kitten-tts-mini-0-8.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "kokoro-82m",
    "bench": "paradee-wikitext200-whisper-base-wer",
    "score": 0.057,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'Kokoro-82M, teacher (af_heart)', column WER = 5.7% (Whisper base transcripts vs source text, §5). Stored as a fraction. Teacher model; reference [11] is 'hexgrad, Kokoro-82M, Hugging Face model card, v1.0' (catalogue kokoro-82m, voice af_heart). Paradee's author is not Kokoro's maker, so by=independent.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "piper-en-us-lessac-medium",
    "bench": "paradee-wikitext200-whisper-base-wer",
    "score": 0.088,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'Piper, en_US-lessac-medium', column WER = 8.8% (Whisper base transcripts vs source text, §5). Stored as a fraction. Voice en_US-lessac-medium (rhasspy/piper-voices; catalogue piper-en-us-lessac-medium); 15.7M params, 63.2 MB fp32 as printed; timed from text.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "kitten-tts-nano-0-8",
    "bench": "paradee-wikitext200-whisper-base-wer",
    "score": 0.055,
    "source": "https://arxiv.org/html/2610.06817v1",
    "by": "independent",
    "date": "2026-10-05",
    "note": "Paradee (Mahendrakar, arXiv:2610.06817v1, 5 Oct 2026, CC BY 4.0) Table 3 'Paradee against other small open models on the same 200 held-out sentences, with the same scoring' (sentences held out from a 12,000-sentence WikiText-103 corpus; each model speaks its own voice). The author ran every system on one Apple M4 Pro machine (§5). Row 'KittenTTS nano 0.8 (Bella)', column WER = 5.5% (Whisper base transcripts vs source text, §5). Stored as a fraction. Proposed comparison-only model kitten-tts-nano-0-8: printed '14.0M' params and '56.8 MB' fp32 file match KittenML/kitten-tts-nano-0.8-fp32 kitten_tts_nano_v0_8.onnx (56,767,095 bytes @7a1db64); voice Bella. Not the catalogue's kitten-tts-mini-0-8.",
    "source_id": "speech-paradee-report-ccby"
  },
  {
    "model": "openaudio-s1-mini",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 931.4,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) entry rank 24, model_id fish_speech 'Fish Speech S1-mini', model_url https://huggingface.co/fishaudio/s1-mini (= catalogue openaudio-s1-mini): elo 931.4 ±23.8, 506 battles (W109/L178/T219), retired: true (retired 2026-05-08 per the Space news). Retired entries are NOT frozen: the backend's elo.py 'compute_leaderboard' docstring says 'Fit a Bradley-Terry model to all votes' and only rank computation skips retired ids; this entry's rating moves between daily snapshots with battles fixed at 506 (965.5 on 2026-05-10 @dcef062, 963.7 on 06-02 @fb3b88b, 940.6 on 07-30 @1a26392, 930.1 on 09-05 @a3332e6, 931.4 on 09-24 @6e003a5), so it is on the same scale as the active rows of this snapshot. Requires relaxing the benchmark's 'only active models' rule (see md).",
    "source_id": "speech-arabic-tts-arena-apache"
  },
  {
    "model": "whisper-medium",
    "bench": "librispeech-test-clean-wer",
    "score": 0.028,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/owsm_v3.1/s2t1/README.md",
    "by": "independent",
    "date": "2024-01-08",
    "note": "ESPnet repository @b902372 (Apache-2.0), egs2/owsm_v3.1/s2t1/README.md 'Results' table (added 2024-01-08, commit c77d583): 'Here are some of the ASR results (WER or CER) with attention-based greedy search.' Column 'Whisper medium (769M)' = openai/whisper-medium (multilingual, not .en; catalogue whisper-medium). ESPnet ran Whisper itself: OWSM paper (arXiv:2309.13876v3, used only as provenance evidence) states 'To ensure fair comparison, we prepare all test data in ESPnet and evaluate Whisper in the same setup instead of reporting results from their paper', and the OWSM v3.1 paper (arXiv:2401.16658v3) says it follows that setup ('greedy search and apply the Whisper text normalizer before scoring'). Proposed as a second file of the existing ESPnet evaluator (speech-espnet-librispeech-20260929) so the 11 ESPnet LibriSpeech recipe models share a run with a main-scale model; protocol differs from the recipe rows (greedy decoding + Whisper normaliser vs recipe beam search/LM), see md. Row 'LibriSpeech test-clean | eng', Whisper medium = 2.8 (WER %). Stored 0.028.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "whisper-medium",
    "bench": "librispeech-test-other-wer",
    "score": 0.065,
    "source": "https://github.com/espnet/espnet/blob/b90237261607e61f5e80b4bb8c362f80b209da3c/egs2/owsm_v3.1/s2t1/README.md",
    "by": "independent",
    "date": "2024-01-08",
    "note": "ESPnet repository @b902372 (Apache-2.0), egs2/owsm_v3.1/s2t1/README.md 'Results' table (added 2024-01-08, commit c77d583): 'Here are some of the ASR results (WER or CER) with attention-based greedy search.' Column 'Whisper medium (769M)' = openai/whisper-medium (multilingual, not .en; catalogue whisper-medium). ESPnet ran Whisper itself: OWSM paper (arXiv:2309.13876v3, used only as provenance evidence) states 'To ensure fair comparison, we prepare all test data in ESPnet and evaluate Whisper in the same setup instead of reporting results from their paper', and the OWSM v3.1 paper (arXiv:2401.16658v3) says it follows that setup ('greedy search and apply the Whisper text normalizer before scoring'). Proposed as a second file of the existing ESPnet evaluator (speech-espnet-librispeech-20260929) so the 11 ESPnet LibriSpeech recipe models share a run with a main-scale model; protocol differs from the recipe rows (greedy decoding + Whisper normaliser vs recipe beam search/LM), see md. Row 'LibriSpeech test-other | eng', Whisper medium = 6.5 (WER %). Stored 0.065.",
    "source_id": "speech-espnet-librispeech-20260929"
  },
  {
    "model": "tacotron2-hifigan-vctk-vits-study",
    "bench": "vits2021-vctk-mos",
    "score": 0.535,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "date": "2021-07-18",
    "note": "VITS (Kim, Kong & Son, ICML 2021, PMLR v139; CC BY 4.0 per the PMLR publication agreement) Table 3 'Comparison of evaluated MOS with 95% confidence intervals on the VCTK dataset'. §4.2: 'We trained the models on the VCTK dataset. ... For Tacotron 2, we broadcasted speaker embedding and concatenated it with the encoder output, and for Glow-TTS, we applied the global conditioning ... The evaluation method is the same as that described in Section 4.1' (crowd-sourced 5-point naturalness MOS, raters hear each sample once, loudness-normalised). Same listening test as the existing vits-vctk row (4.38). Stored (MOS-1)/4. Row 'Tacotron 2 + HiFi-GAN', MOS 3.14 (±0.09). Multi-speaker Tacotron 2 (NVIDIA implementation, §3.4 footnote 3) trained on VCTK by the VITS authors, HiFi-GAN vocoder without fine-tuning.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "glow-tts-hifigan-vctk-vits-study",
    "bench": "vits2021-vctk-mos",
    "score": 0.69,
    "source": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf",
    "by": "independent",
    "date": "2021-07-18",
    "note": "VITS (Kim, Kong & Son, ICML 2021, PMLR v139; CC BY 4.0 per the PMLR publication agreement) Table 3 'Comparison of evaluated MOS with 95% confidence intervals on the VCTK dataset'. §4.2: 'We trained the models on the VCTK dataset. ... For Tacotron 2, we broadcasted speaker embedding and concatenated it with the encoder output, and for Glow-TTS, we applied the global conditioning ... The evaluation method is the same as that described in Section 4.1' (crowd-sourced 5-point naturalness MOS, raters hear each sample once, loudness-normalised). Same listening test as the existing vits-vctk row (4.38). Stored (MOS-1)/4. Row 'Glow-TTS + HiFi-GAN', MOS 3.76 (±0.07). Multi-speaker Glow-TTS (jaywalnut310/glow-tts implementation, global speaker conditioning) trained on VCTK by the VITS authors, HiFi-GAN vocoder without fine-tuning.",
    "source_id": "speech-vits-icml2021"
  },
  {
    "model": "speechbrain-asr-wav2vec2-librispeech",
    "bench": "librispeech-test-clean-wer",
    "score": 0.019,
    "source": "https://huggingface.co/speechbrain/asr-wav2vec2-librispeech/blob/356e347b73885161e8996d9a79b6588dc8bf9f9b/README.md",
    "by": "lab",
    "date": "2024-02-19",
    "note": "SpeechBrain card speechbrain/asr-wav2vec2-librispeech @356e347 (license: apache-2.0): 'The performance of the model is the following' table, row 'Release 24-03-22', 'Test clean WER' = 1.90; front-matter model-index 'LibriSpeech (clean) ... Test WER value: 1.90'. wav2vec2-large-960h-lv60-self + 2 DNN layers + CTC, no LM. Differs from PERFORMANCE.md's recipe row (1.65/3.67), so the card value for the released checkpoint is used. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "speechbrain-asr-wav2vec2-librispeech",
    "bench": "librispeech-test-other-wer",
    "score": 0.0396,
    "source": "https://huggingface.co/speechbrain/asr-wav2vec2-librispeech/blob/356e347b73885161e8996d9a79b6588dc8bf9f9b/README.md",
    "by": "lab",
    "date": "2024-02-19",
    "note": "SpeechBrain card speechbrain/asr-wav2vec2-librispeech @356e347 (apache-2.0): performance table row 'Release 24-03-22', 'Test other WER' = 3.96; model-index 'LibriSpeech (other) ... Test WER value: 3.96'. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "speechbrain-asr-conformer-largescaleasr",
    "bench": "librispeech-test-other-wer",
    "score": 0.046,
    "source": "https://huggingface.co/speechbrain/asr-conformer-loquacious/blob/42106e67e454ae361ca2d8311a7a807d5f95a3b9/README.md",
    "by": "lab",
    "date": "2025-02-06",
    "note": "SpeechBrain card speechbrain/asr-conformer-loquacious @42106e6 (formerly asr-conformer-largescaleasr; apache-2.0; 'This model is a contribution of the Samsung AI Center-Cambridge'): 'The performance of the model is the following' table, row '480M', column 'LibriSpeech test-other' = 4.6 (zero-shot: the model is trained on LargeScaleASR/Loquacious, not LibriSpeech). Conformer encoder + joint CTC/transformer decoder, no external LM. Results added 2025-02-06 (commit 82b04aa 'more results'); later commits README only. The card prints no test-clean value. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "speechbrain-asr-conformer-largescaleasr",
    "bench": "voxpopuli-en-wer",
    "score": 0.069,
    "source": "https://huggingface.co/speechbrain/asr-conformer-loquacious/blob/42106e67e454ae361ca2d8311a7a807d5f95a3b9/README.md",
    "by": "lab",
    "date": "2025-02-06",
    "note": "Same card and table, column 'Voxpopuli test' = 6.9 (English VoxPopuli test). Raw evidence only: no other SpeechBrain card in this pool reports VoxPopuli, so this run has one model. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "speechbrain-asr-transformer-transformerlm-librispeech",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0227,
    "source": "https://huggingface.co/speechbrain/asr-transformer-transformerlm-librispeech/blob/bcf56f762f37877912578f8800bc2a4f5171e5e5/README.md",
    "by": "lab",
    "date": "2024-02-19",
    "note": "SpeechBrain card speechbrain/asr-transformer-transformerlm-librispeech @bcf56f7 (apache-2.0): performance table row 'Release 24-03-22', 'Test clean WER' = 2.27; model-index Test WER 2.27 (also PERFORMANCE.md row transformer.yaml 2.27%). Transformer encoder + joint CTC/transformer decoder with Transformer LM. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "speechbrain-asr-transformer-transformerlm-librispeech",
    "bench": "librispeech-test-other-wer",
    "score": 0.0553,
    "source": "https://huggingface.co/speechbrain/asr-transformer-transformerlm-librispeech/blob/bcf56f762f37877912578f8800bc2a4f5171e5e5/README.md",
    "by": "lab",
    "date": "2024-02-19",
    "note": "Same card, 'Test other WER' = 5.53; model-index Test WER 5.53. Pooled with the other SpeechBrain cards under first-party-speechbrain-asr-cards (one maker's own published results, like first-party-meta-asr-cards).",
    "source_id": "first-party-speechbrain-asr-cards"
  },
  {
    "model": "kokoro-82m",
    "bench": "carun-tts-bench56-whisper-v3-wer",
    "score": 0.023,
    "source": "https://github.com/carun/tts-bench/blob/27727b24abbc688f718992eb5bfc5c7121facfcb/data/REPORT_DATA.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "carun/tts-bench @27727b2 (Apache-2.0 repository, committed 2026-09-10 'data: raw results from the published run'), data/REPORT_DATA.md 'Intelligibility — Whisper judge' table, column 'Overall WER' (same value in README's 'WER (Whisper judge)' column). 56 fixed English sentences in 7 categories (harness/corpus.py), one run per engine on an NVIDIA GB10; transcribed with openai/whisper-large-v3 and scored after whisper_normalizer EnglishTextNormalizer on both sides (harness/judge.py). The repo author ran every engine. Row 'kokoro' = 2.3%. engines/run_kokoro.py: kokoro KPipeline (hexgrad/Kokoro-82M v1.0), VOICE = 'af_heart', speed 1.",
    "source_id": "speech-carun-tts-bench-apache"
  },
  {
    "model": "magpie-tts-multilingual-357m",
    "bench": "carun-tts-bench56-whisper-v3-wer",
    "score": 0.07,
    "source": "https://github.com/carun/tts-bench/blob/27727b24abbc688f718992eb5bfc5c7121facfcb/data/REPORT_DATA.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "carun/tts-bench @27727b2 (Apache-2.0 repository, committed 2026-09-10 'data: raw results from the published run'), data/REPORT_DATA.md 'Intelligibility — Whisper judge' table, column 'Overall WER' (same value in README's 'WER (Whisper judge)' column). 56 fixed English sentences in 7 categories (harness/corpus.py), one run per engine on an NVIDIA GB10; transcribed with openai/whisper-large-v3 and scored after whisper_normalizer EnglishTextNormalizer on both sides (harness/judge.py). The repo author ran every engine. Row 'magpie' = 7.0% (default configuration, APPLY_TN = False; the separate 'magpie-tn' text-normalisation run, 5.9%, is not recorded). engines/run_magpie.py: MagpieTTSModel.from_pretrained('nvidia/magpie_tts_multilingual_357m'), language 'en', speaker_index 4 (Sofia).",
    "source_id": "speech-carun-tts-bench-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-customvoice",
    "bench": "carun-tts-bench56-whisper-v3-wer",
    "score": 0.041,
    "source": "https://github.com/carun/tts-bench/blob/27727b24abbc688f718992eb5bfc5c7121facfcb/data/REPORT_DATA.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "carun/tts-bench @27727b2 (Apache-2.0 repository, committed 2026-09-10 'data: raw results from the published run'), data/REPORT_DATA.md 'Intelligibility — Whisper judge' table, column 'Overall WER' (same value in README's 'WER (Whisper judge)' column). 56 fixed English sentences in 7 categories (harness/corpus.py), one run per engine on an NVIDIA GB10; transcribed with openai/whisper-large-v3 and scored after whisper_normalizer EnglishTextNormalizer on both sides (harness/judge.py). The repo author ran every engine. Row 'qwen3-tts-0.6b' = 4.1%. engines/run_qwen.py: MODEL = 'Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice', SPEAKER = 'Aiden', generate_custom_voice(language='English'), sdpa attention.",
    "source_id": "speech-carun-tts-bench-apache"
  },
  {
    "model": "wayu-paxa-tts-edge",
    "bench": "wayu-libritts500-en-whisper-v3-cer",
    "score": 0.011,
    "source": "https://arxiv.org/html/2609.03502v1",
    "by": "lab",
    "date": "2026-09-03",
    "note": "Building and Evaluating Fixed-Voice Thai TTS from Synthetic Speech (arXiv:2609.03502v1, CC BY 4.0; existing ledger entry speech-wayu-thai-tts-ccby) Table 8 'Final comparison', column 'CER Eng.': caption 'English CER uses 500 held-out LibriTTS lines transcribed with Whisper Large V3'. Same table and run as the recorded Thai CER and keyword-accuracy rows. Stored as a fraction. Row 'Wayu-Paxa-TTS-Edge' = 1.1%.",
    "source_id": "speech-wayu-thai-tts-ccby"
  },
  {
    "model": "omnivoice",
    "bench": "wayu-libritts500-en-whisper-v3-cer",
    "score": 0.009,
    "source": "https://arxiv.org/html/2609.03502v1",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Building and Evaluating Fixed-Voice Thai TTS from Synthetic Speech (arXiv:2609.03502v1, CC BY 4.0; existing ledger entry speech-wayu-thai-tts-ccby) Table 8 'Final comparison', column 'CER Eng.': caption 'English CER uses 500 held-out LibriTTS lines transcribed with Whisper Large V3'. Same table and run as the recorded Thai CER and keyword-accuracy rows. Stored as a fraction. Row 'OmniVoice teacher' = 0.9%.",
    "source_id": "speech-wayu-thai-tts-ccby"
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "wayu-libritts500-en-whisper-v3-cer",
    "score": 0.008,
    "source": "https://arxiv.org/html/2609.03502v1",
    "by": "independent",
    "date": "2026-09-03",
    "note": "Building and Evaluating Fixed-Voice Thai TTS from Synthetic Speech (arXiv:2609.03502v1, CC BY 4.0; existing ledger entry speech-wayu-thai-tts-ccby) Table 8 'Final comparison', column 'CER Eng.': caption 'English CER uses 500 held-out LibriTTS lines transcribed with Whisper Large V3'. Same table and run as the recorded Thai CER and keyword-accuracy rows. Stored as a fraction. Row 'Gemini 3.1 Flash TTS' = 0.8%.",
    "source_id": "speech-wayu-thai-tts-ccby"
  },
  {
    "model": "qwen-image-2512",
    "bench": "ultratext-bench-composite",
    "score": 0.6796,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Qwen-Image-2512', column 'Composite' = 67.96 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "z-image-base",
    "bench": "ultratext-bench-composite",
    "score": 0.6192,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Z-Image-Base', column 'Composite' = 61.92 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "z-image-turbo",
    "bench": "ultratext-bench-composite",
    "score": 0.5438,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Z-Image-Turbo', column 'Composite' = 54.38 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "hidream-o1-image",
    "bench": "ultratext-bench-composite",
    "score": 0.4556,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'HiDream-O1-Image', column 'Composite' = 45.56 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "llada-image",
    "bench": "ultratext-bench-composite",
    "score": 0.4392,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "lab",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'LLaDA-Image', column 'Composite' = 43.92 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). by=lab: the UltraText authors (Westlake University LINs Lab / Ant Group) overlap with the LLaDA-Image authors (Deyuan Liu, Jun Xie, Jiacheng Liu, Zhenglin Cheng, Peng Sun, Tao Lin).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "llada-image-turbo",
    "bench": "ultratext-bench-composite",
    "score": 0.4129,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "lab",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'LLaDA-Image-Turbo', column 'Composite' = 41.29 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). by=lab: the UltraText authors (Westlake University LINs Lab / Ant Group) overlap with the LLaDA-Image authors (Deyuan Liu, Jun Xie, Jiacheng Liu, Zhenglin Cheng, Peng Sun, Tao Lin).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "qwen-image",
    "bench": "ultratext-bench-composite",
    "score": 0.4127,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Qwen-Image', column 'Composite' = 41.27 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "flux-2-dev",
    "bench": "ultratext-bench-composite",
    "score": 0.4103,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'FLUX.2 [Dev]', column 'Composite' = 41.03 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "ultratext-bench-composite",
    "score": 0.2993,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Hunyuan-Image-3.0', column 'Composite' = 29.93 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). Listed under open-weight models, so the base HunyuanImage-3.0 weights, not -Instruct.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "flux-2-klein-9b",
    "bench": "ultratext-bench-composite",
    "score": 0.2294,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'FLUX.2 [Klein] 9B', column 'Composite' = 22.94 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). Distilled [klein] 9B (the separate -Base checkpoints are labelled Base elsewhere).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "flux-2-klein-4b",
    "bench": "ultratext-bench-composite",
    "score": 0.1958,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'FLUX.2 [Klein] 4B', column 'Composite' = 19.58 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). Distilled [klein] 4B.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "flux-1-dev",
    "bench": "ultratext-bench-composite",
    "score": 0.1817,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'FLUX.1 [Dev]', column 'Composite' = 18.17 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "sd-3-5-large",
    "bench": "ultratext-bench-composite",
    "score": 0.0748,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'SD3.5 Large', column 'Composite' = 7.48 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "gpt-image-2",
    "bench": "ultratext-bench-composite",
    "score": 0.9935,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'GPT Image 2 [Low]', column 'Composite' = 99.35 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). API quality setting 'Low' (bracketed label); same model id, quality tier retained in note.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "nano-banana-2",
    "bench": "ultratext-bench-composite",
    "score": 0.97,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Nano Banana 2', column 'Composite' = 97.00 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "seedream-5-0-pro",
    "bench": "ultratext-bench-composite",
    "score": 0.9145,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Seedream 5.0 Pro', column 'Composite' = 91.45 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "qwen-image-2-0-pro",
    "bench": "ultratext-bench-composite",
    "score": 0.9021,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Qwen-Image-2.0-Pro', column 'Composite' = 90.21 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). Release label 'Qwen-Image-2.0-Pro' retained; no API snapshot inferred (same mapping as the existing Qwen-Image-Bench, FEPBench and ChartSync rows).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "qwen-image-3-0-pro",
    "bench": "ultratext-bench-composite",
    "score": 0.8833,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Qwen-Image-3.0', column 'Composite' = 88.33 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). Mapping: the paper's API-only row is labelled 'Qwen-Image-3.0'; Alibaba Model Studio documents a single Qwen-Image 3.0 API model, qwen-image-3.0-pro (https://www.alibabacloud.com/help/en/model-studio/qwen-image-3-0-pro, checked 2026-10-08), so the row is mapped to qwen-image-3-0-pro. Review flag.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "wan-2-7-image-pro",
    "bench": "ultratext-bench-composite",
    "score": 0.8573,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'WAN-2.7-Image-Pro', column 'Composite' = 85.73 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G).",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "ultratext-bench-composite",
    "score": 0.5796,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'GPT Image 1.5 [High]', column 'Composite' = 57.96 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). API quality setting 'High'.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "gpt-image-1",
    "bench": "ultratext-bench-composite",
    "score": 0.3791,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'GPT Image 1 [High]', column 'Composite' = 37.91 (0-100 VLM rating; stored /100). Composite = 60/30/5/5 weighted Fidelity/Clarity/Spatial/Scene, averaged equally over the English and Chinese prompt-macro means (432 prompts: 24 scene categories x 3 difficulty levels, 216 EN + 216 ZH; 4-12 exact-string text regions per prompt); judge Q-Judger (from Qwen-Image-Bench) scoring each image against the full region reference. Authors' own run: Sec. 5.1 'We evaluated 24 model configurations ... listed in Table 5. ... Each model uses its standard sampling configuration; resolution, aspect ratio, and API behavior can differ.' Model revisions/dates not published (Appendix G). API quality setting 'High'.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "aime-2024",
    "score": 0.833,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'AIME 2024' = 83.3. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "aime-2025",
    "score": 0.746,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'AIME 2025' = 74.6. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "math-500",
    "score": 0.96,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'MATH-500' = 96.0. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.623,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'LiveCodeBench *(24/8~25/5)*' = 62.3. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "gpqa-diamond",
    "score": 0.692,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'GPQA Diamond' = 69.2. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "hle",
    "score": 0.072,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'HLE *(no tools)*' = 7.2\\*. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5. Marker '*': conducted on the text-only HLE subset.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "zebralogic",
    "score": 0.801,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'ZebraLogic' = 80.1. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "mmlu-pro",
    "score": 0.806,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'MMLU-Pro' = 80.6. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "swe-bench-verified",
    "score": 0.556,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'SWE-bench Verified' = 55.6. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "longbench-v2",
    "score": 0.61,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'LongBench-v2' = 61.0. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "tau-bench-airline-gpt41-user",
    "score": 0.6,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'TAU-bench *(airline)*' = 60.0. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "tau-bench-retail-gpt41-user",
    "score": 0.678,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'TAU-bench *(retail)*' = 67.8. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "simpleqa",
    "score": 0.179,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'SimpleQA' = 17.9. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "minimax-m1-40k",
    "bench": "multichallenge",
    "score": 0.447,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md",
    "by": "lab",
    "date": "2025-06-16",
    "note": "MiniMax own evaluation in the pinned MiniMax-M1-80k model card, '2. Evaluation' table, column 'MiniMax-M1-40K' (extended thinking 40K), row 'MultiChallenge' = 44.7. Same card, harness and settings as the existing MiniMax-M1-80K rows: temperature=1.0, top_p=0.95; SWE-bench Verified via MiniMax's Agentless-derived scaffold on n=486 tasks; TAU-bench with GPT-4.1 as user model, no custom tools, max 40 steps; LiveCodeBench window 24/8-25/5.",
    "source_id": "card-minimaxai--minimax-m1-80k"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "bbh",
    "score": 0.7583,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'BBH' (3-shot, CoT), section 'General Tasks' = 75.83. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "bbh",
    "score": 0.8374,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'BBH' (3-shot, CoT), section 'General Tasks' = 83.74. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "bfcl-v3",
    "score": 0.608,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'BFCL v3', section 'Tool Calling Tasks' = 60.80. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "bfcl-v3",
    "score": 0.7368,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'BFCL v3', section 'Tool Calling Tasks' = 73.68. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "bigcodebench",
    "score": 0.3219,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'BigCodeBench' (pass@1), section 'Code Tasks' = 32.19. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "bigcodebench",
    "score": 0.3877,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'BigCodeBench' (pass@1), section 'Code Tasks' = 38.77. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "cruxeval-o",
    "score": 0.4075,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 40.75. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "cruxeval-o",
    "score": 0.5575,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 55.75. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "deepmind-math",
    "score": 0.6464,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 64.64. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "deepmind-math",
    "score": 0.8193,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 81.93. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "gpqa-unspecified",
    "score": 0.317,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'GPQA' (0-shot, CoT), section 'General Tasks' = 31.70. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "gpqa-unspecified",
    "score": 0.4576,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'GPQA' (0-shot, CoT), section 'General Tasks' = 45.76. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "gsm-symbolic",
    "score": 0.8132,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 81.32. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "gsm-symbolic",
    "score": 0.757,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 75.70. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "gsm8k",
    "score": 0.8688,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'GSM8K' (8-shot), section 'Math Tasks' = 86.88. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "gsm8k",
    "score": 0.9416,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'GSM8K' (8-shot), section 'Math Tasks' = 94.16. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "humaneval",
    "score": 0.8171,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'HumanEval' (pass@1), section 'Code Tasks' = 81.71. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "humaneval",
    "score": 0.8841,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'HumanEval' (pass@1), section 'Code Tasks' = 88.41. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "humaneval-plus",
    "score": 0.7683,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'HumanEval+' (pass@1), section 'Code Tasks' = 76.83. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "humaneval-plus",
    "score": 0.8537,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'HumanEval+' (pass@1), section 'Code Tasks' = 85.37. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "ifeval-unspecified",
    "score": 0.823,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'IFEval Avg', section 'Alignment Tasks' = 82.30. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "ifeval-unspecified",
    "score": 0.8965,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'IFEval Avg', section 'Alignment Tasks' = 89.65. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "include",
    "score": 0.5205,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 52.05. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "include",
    "score": 0.6726,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 67.26. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mbpp",
    "score": 0.7116,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MBPP' (pass@1), section 'Code Tasks' = 71.16. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mbpp",
    "score": 0.8545,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MBPP' (pass@1), section 'Code Tasks' = 85.45. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mbpp-plus",
    "score": 0.6217,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MBPP+' (pass@1), section 'Code Tasks' = 62.17. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mbpp-plus",
    "score": 0.7354,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MBPP+' (pass@1), section 'Code Tasks' = 73.54. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mgsm-5lang",
    "score": 0.7,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MGSM' (8-shot), section 'Multilingual Tasks' = 70.00. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mgsm-5lang",
    "score": 0.7112,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MGSM' (8-shot), section 'Multilingual Tasks' = 71.12. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "minerva-math",
    "score": 0.6794,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 67.94. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "minerva-math",
    "score": 0.8132,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 81.32. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mmlu",
    "score": 0.6702,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MMLU' (5-shot), section 'General Tasks' = 67.02. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mmlu",
    "score": 0.8016,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MMLU' (5-shot), section 'General Tasks' = 80.16. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mmlu-pro",
    "score": 0.4983,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 49.83. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mmlu-pro",
    "score": 0.6409,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 64.09. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "mmmlu",
    "score": 0.5761,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 57.61. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "mmmlu",
    "score": 0.7371,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 73.71. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "multipl-e",
    "score": 0.5254,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'MULTIPLE' (pass@1), section 'Code Tasks' = 52.54. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "multipl-e",
    "score": 0.6231,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'MULTIPLE' (pass@1), section 'Code Tasks' = 62.31. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-3b",
    "bench": "simpleqa",
    "score": 0.0368,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '3B Dense' column, row 'SimpleQA', section 'General Tasks' = 3.68. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "granite-4-1-30b",
    "bench": "simpleqa",
    "score": 0.0681,
    "source": "https://huggingface.co/ibm-granite/granite-4.1-8b/blob/1504002f650e656a0a3789d99574df12e3e94ed0/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-05-04",
    "note": "IBM own evaluation in the pinned granite-4.1-8b model card, Evaluation Results table, '30B Dense' column, row 'SimpleQA', section 'General Tasks' = 6.81. Same table, column mapping and exclusions as the existing granite-4-1-8b rows ('8B Dense' column): card language table MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; IFEval row is the card's 'IFEval Avg'.",
    "source_id": "card2-ibm-granite-granite-4-1-8b"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "mmlu",
    "score": 0.76,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'MMLU' = 76.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "mmlu-pro",
    "score": 0.66,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'MMLU-Pro' = 66.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "c-eval",
    "score": 0.854,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'C-Eval' = 85.4 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "cmmlu",
    "score": 0.848,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'CMMLU' = 84.8 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "chinese-simpleqa",
    "score": 0.489,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'ChineseSimpleQA' = 48.9 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "simpleqa",
    "score": 0.242,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'SimpleQA' = 24.2 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.8,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'IFEval' = 80.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "multi-if-unspecified",
    "score": 0.61,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'Multi-IF' = 61.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "sysbench-unspecified",
    "score": 0.48,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'Sysbench' = 48.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "math-500",
    "score": 0.872,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'MATH-500' = 87.2 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "gsm8k",
    "score": 0.933,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'GSM8K' = 93.3 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "aime-2024",
    "score": 0.279,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'AIME'24' = 27.9 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "aime-2025",
    "score": 0.197,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'AIME'25' = 19.7 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "bbh",
    "score": 0.83,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'BBH' = 83.0 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "musr",
    "score": 0.677,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'MUSR' = 67.7 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "livecodebench-unversioned",
    "score": 0.264,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'LiveCodeBench' = 26.4 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "humaneval-plus",
    "score": 0.896,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'Humaneval+' = 89.6 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "mbpp-plus",
    "score": 0.775,
    "source": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Baidu own evaluation in the pinned ERNIE README, section 'Performance of post-trained model ERNIE-4.5-21B-A3B' (chart image https://github.com/user-attachments/assets/5bacaae8-ef27-494d-8c65-589ba187a084, sha256 2607686116a5ecb5f2f8592841514d6c3c763c783c9336a92d1cc635af8fcb37, fetched 2026-10-08), column 'ERNIE-4.5-21B-A3B', row 'MBPP+' = 77.5 (printed label). Same README, benchmark labels and mapping as the existing ERNIE-4.5-300B-A47B rows (post-trained models; shot counts and generation settings not stated; LCB window and IF aggregation unspecified). The chart's Qwen3-30B-A3B column is a competitor and is not used.",
    "source_id": "card-ernie-45-posttrained-final"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "ifeval-unspecified",
    "score": 0.7944,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'IFEval' = 79.44. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "ifbench-unspecified",
    "score": 0.26,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'IFBench' = 26.00. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "multi-if-unspecified",
    "score": 0.5854,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'Multi-IF' = 58.54. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "multi-if-unspecified",
    "score": 0.7993,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2.5-8B-A1B', row 'Multi-IF' = 79.93. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "math-500",
    "score": 0.748,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'MATH500' = 74.80. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "aime-2025",
    "score": 0.2,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'AIME25' = 20.00. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "bfcl-v3",
    "score": 0.4507,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'BFCLv3' = 45.07. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "bfcl-v3",
    "score": 0.6436,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2.5-8B-A1B', row 'BFCLv3' = 64.36. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used. The card's 'Math and agentic workflows' table prints 64.79 for the same model; the 64.36 from the predecessor-comparison table is recorded so both LFM rows come from one table.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "bfcl-v4",
    "score": 0.2552,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'BFCLv4' = 25.52. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "bfcl-v4",
    "score": 0.485,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2.5-8B-A1B', row 'BFCLv4' = 48.50. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used. The 'Math and agentic workflows' table prints 49.73; 48.50 from the predecessor-comparison table is recorded for consistency with the LFM2 row.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "tau2-telecom",
    "score": 0.136,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'Tau² Telecom' = 13.60. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "tau2-retail",
    "score": 0.0702,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Liquid AI own evaluation in the pinned LFM2.5-8B-A1B card, table 'Improvements over LFM2-8B-A1B', column 'LFM2-8B-A1B', row 'Tau² Retail' = 7.02. Own predecessor/successor comparison; benchmark versions as in the existing LFM2.5-8B-A1B rows (IFEval/IFBench/Multi-IF aggregation not stated; tau2 domains telecom and retail). The AA-Omniscience rows (Artificial Analysis) and all competitor tables are not used.",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8371,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); cell '77.93 ± 1.72 / 83.71 ± 1.25': score = accuracy 83.71 (EuroEval's secondary metric; primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'baidu/ERNIE-4.5-21B-A3B-PT', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.8298,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); cell '77.33 ± 0.74 / 82.98 ± 0.55': score = accuracy 82.98 (EuroEval's secondary metric; primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'baidu/ERNIE-4.5-21B-A3B-PT', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmlu",
    "score": 0.721,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'MMLU' = 72.1. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmlu-pro",
    "score": 0.495,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'MMLU-Pro' = 49.5. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "agieval-marco-unspecified",
    "score": 0.452,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'AGIEval' = 45.2. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "gpqa-diamond",
    "score": 0.319,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'GPQA-Diamond' = 31.9. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "gsm8k",
    "score": 0.846,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'GSM8K' = 84.6. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "math-subset-unspecified",
    "score": 0.826,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'MATH' = 82.6. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "global-mmlu-marco",
    "score": 0.49,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Acc@1, one response/query. Row 'GlobalMMLU' = 49.0. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmmlu-marco",
    "score": 0.565,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Acc@1, one response/query. Row 'MMMLU' = 56.5. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmlu-prox-lite-marco",
    "score": 0.338,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'MMLU-ProX-Lite' = 33.8. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mgsm-marco",
    "score": 0.565,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'MGSM' = 56.5. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "polymath-marco",
    "score": 0.265,
    "source": "https://arxiv.org/html/2604.25578v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Table 11 own Light-Eval baseline run, column 'LFM2 / 8B-A1B / Instruct' (column 4). Exact LFM2-8B-A1B identified in Section 4.3.1 ('MoE models: Granite4-Tiny & Granite4-Small, and LFM2-8B-A1B & LFM2-24B-A2B'; 'all models are conducted using the Light-Eval framework to ensure fair comparisons'). Temperature 1.0, top_p 1.0, max 8192 tokens. Avg@8 mean across 8 responses/query, not pass@8 or majority voting. Row 'PolyMath' = 26.5. Same mapping and exclusions as the existing LFM2-24B-A2B rows (shot count unspecified; MATH subset not resolved; no MMLU-Redux or averages).",
    "source_id": "report-marco-moe-260425578v1"
  },
  {
    "model": "omnigen2",
    "bench": "creval-bench-overall",
    "score": 0.6817,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'OmniGen2 [49]', column Overall/avg = 68.17 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "uniworld-v1",
    "bench": "creval-bench-overall",
    "score": 0.6537,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'UniWorld-V1 [23]', column Overall/avg = 65.37 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "bagel",
    "bench": "creval-bench-overall",
    "score": 0.6882,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'Bagel [7]', column Overall/avg = 68.82 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'. Default (non-thinking) row; 'Bagel (think)' not used.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "step1x-edit-v1-2",
    "bench": "creval-bench-overall",
    "score": 0.6946,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'Step1X-Edit v1-p2 [27]', column Overall/avg = 69.46 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'. 'Step1X-Edit v1-p2' = the v1p2 release (Step1X-Edit v1.2); the separate '(think)' row is not used.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "creval-bench-overall",
    "score": 0.7481,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'FLUX.1 Kontext [dev] [2]', column Overall/avg = 74.81 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "creval-bench-overall",
    "score": 0.7488,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'FLUX.1 Kontext [pro] [2]', column Overall/avg = 74.88 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "seedream-4-0",
    "bench": "creval-bench-overall",
    "score": 0.8343,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'Seedream 4.0 [38]', column Overall/avg = 83.43 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "nano-banana",
    "bench": "creval-bench-overall",
    "score": 0.8134,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'Gemini 2.5 Flash Image [10]', column Overall/avg = 81.34 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'. 'Gemini 2.5 Flash Image' = Nano Banana.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "gpt-image-1",
    "bench": "creval-bench-overall",
    "score": 0.7897,
    "source": "https://arxiv.org/html/2603.26174v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "CREval-Bench (arXiv 2603.26174v1, Mar 2026) Table 2, row 'GPT-Image-1 [30]', column Overall/avg = 78.97 (stored /100). Overall avg = weighted average of Instruction Following, Visual Consistency and Visual Quality scores over the Customization, Contextualization and Stylization editing types; 800+ creative editing samples, 13K QA evaluation queries, GPT-4o as the QA evaluator. Authors' own run: 'Leveraging this pipeline and benchmark, we systematically evaluate a diverse set of state-of-the-art open and closed-source models' and 'In Stage 3, we evaluate mainstream image manipulation models using CREval method'.",
    "source_id": "paper-2603.26174v1-creval"
  },
  {
    "model": "nano-banana",
    "bench": "servimage-task-acceptance",
    "score": 0.2991,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Gemini-Banana' (API endpoint 'gemini-2.5-flash-image' per Appendix Table 6), column 'Task Acc. (%)' = 29.91 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "gpt-image-1",
    "bench": "servimage-task-acceptance",
    "score": 0.4822,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'GPT-Image-1' (API endpoint 'gpt-image-1' per Appendix Table 6), column 'Task Acc. (%)' = 48.22 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9. Sec. 5.1 calls the family 'GPT-5-Image'; the table label and API endpoint are gpt-image-1.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "gpt-image-1-mini",
    "bench": "servimage-task-acceptance",
    "score": 0.3236,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'GPT-Image-1-Mini' (API endpoint 'gpt-image-1-mini' per Appendix Table 6), column 'Task Acc. (%)' = 32.36 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9. Sec. 5.1 calls it 'gpt-5-image-mini'; the table label and API endpoint are gpt-image-1-mini.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "ideogram-3",
    "bench": "servimage-task-acceptance",
    "score": 0.2589,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Ideogram-v3' (API endpoint 'ideogram-v3-text-to-image' per Appendix Table 6), column 'Task Acc. (%)' = 25.89 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9. Quality/speed tier not stated.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "imagen-4",
    "bench": "servimage-task-acceptance",
    "score": 0.3103,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Imagen-4.0' (API endpoint 'imagen4' per Appendix Table 6), column 'Task Acc. (%)' = 31.03 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "seededit-3-0",
    "bench": "servimage-task-acceptance",
    "score": 0.0698,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'SeedEdit-3.0-i2i' (API endpoint 'seededit-3.0-i2i' per Appendix Table 6), column 'Task Acc. (%)' = 6.98 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "seedream-3-0",
    "bench": "servimage-task-acceptance",
    "score": 0.314,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Seedream-3.0-t2i' (API endpoint 'seedream-3.0-t2i' per Appendix Table 6), column 'Task Acc. (%)' = 31.40 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "seedream-4-0",
    "bench": "servimage-task-acceptance",
    "score": 0.3112,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Seedream-4.0' (API endpoint 'seedream-4.0' per Appendix Table 6), column 'Task Acc. (%)' = 31.12 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "flux-1-1-pro",
    "bench": "servimage-task-acceptance",
    "score": 0.1,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'FLUX-1.1-Pro' (API endpoint 'flux-1.1-pro' per Appendix Table 6), column 'Task Acc. (%)' = 10.00 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "servimage-task-acceptance",
    "score": 0.1647,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'FLUX-Kontext-Pro' (API endpoint 'flux-kontext-pro' per Appendix Table 6), column 'Task Acc. (%)' = 16.47 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "qwen-image",
    "bench": "servimage-task-acceptance",
    "score": 0.2495,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'Qwen-Image' (API endpoint 'qwen-text-to-image' per Appendix Table 6), column 'Task Acc. (%)' = 24.95 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "sd-3-5-large",
    "bench": "servimage-task-acceptance",
    "score": 0.086,
    "source": "https://arxiv.org/html/2604.24023v2",
    "by": "independent",
    "date": "2026-06-19",
    "note": "ServImage (arXiv 2604.24023v2) Table 2, row 'SD-3.5-Large' (API endpoint 'stable-diffusion-3.5-large' per Appendix Table 6), column 'Task Acc. (%)' = 8.60 (stored /100): share of the 1,070 paid commercial design tasks (portrait, e-commerce product, digital content; T2I and editing; $295k contract value) whose deliverables were approved under the original contract according to human payment-decision labels, standard settlement scenario. Authors' own run: Sec. 3.4 'we run the 16 image generation and editing models introduced in Sec. 5 ... on all paid tasks', Sec. 5.1 'We evaluate 16 image models'. Human labels, not the ServImageModel predictions of Table 9.",
    "source_id": "paper-2604.24023v2-servimage"
  },
  {
    "model": "internvl2-26b",
    "bench": "docvqa",
    "score": 0.929,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'DocVQA test' = 92.9 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "chartqa",
    "score": 0.849,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'ChartQA test' = 84.9 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "textvqa",
    "score": 0.823,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'TextVQA val' = 82.3 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "ocrbench",
    "score": 0.825,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'OCRBench' = 825 (divided by 1000). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "realworldqa",
    "score": 0.683,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'RealWorldQA' = 68.3 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "ai2d-test",
    "score": 0.845,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'AI2D test' = 84.5 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "mmmu",
    "score": 0.512,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'MMMU val' = 51.2 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "hallusionbench",
    "score": 0.507,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'HallBench avg' = 50.7 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25. 'HallBench avg' (HallusionBench average accuracy).",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "mathvista-mini",
    "score": 0.594,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'MathVista testmini' = 59.4 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "mvbench",
    "score": 0.675,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'MVBench' = 67.5 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "videomme-without-sub",
    "score": 0.548,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'Video-MME w/o subs' = 54.8 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-26b",
    "bench": "videomme-with-sub",
    "score": 0.571,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-26B/blob/46f37972e05604cc44a59f72334df220fa197a7b/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-26B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-26B', row 'Video-MME w subs' = 57.1 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "docvqa",
    "score": 0.892,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'DocVQA test' = 89.2 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "chartqa",
    "score": 0.815,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'ChartQA test' = 81.5 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "textvqa",
    "score": 0.744,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'TextVQA val' = 74.4 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "ocrbench",
    "score": 0.788,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'OCRBench' = 788 (divided by 1000). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "realworldqa",
    "score": 0.607,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'RealWorldQA' = 60.7 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "ai2d-test",
    "score": 0.789,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'AI2D test' = 78.9 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "mmmu",
    "score": 0.479,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'MMMU val' = 47.9 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "hallusionbench",
    "score": 0.419,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'HallBench avg' = 41.9 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25. 'HallBench avg' (HallusionBench average accuracy).",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "mathvista-mini",
    "score": 0.586,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'MathVista testmini' = 58.6 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "mvbench",
    "score": 0.637,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'MVBench' = 63.7 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "videomme-without-sub",
    "score": 0.514,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'Video-MME w/o subs' = 51.4 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "internvl2-4b",
    "bench": "videomme-with-sub",
    "score": 0.534,
    "source": "https://huggingface.co/OpenGVLab/InternVL2-4B/blob/6eb21ec278fbdb03b751a3d48d8dfc9724e05608/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-25",
    "note": "OpenGVLab's own evaluation in the pinned InternVL2-4B model card (MIT), 'Evaluation' section ('Image Benchmarks' / 'Video Benchmarks' tables), column 'InternVL2-4B', row 'Video-MME w subs' = 53.4 (divided by 100). Own-model column only; GPT-4T, Gemini-1.5-Pro, Phi-3-Vision, VideoChat2 and InternVL-Chat-V1-5 / Mini-InternVL columns not used. Card: 'We simultaneously use InternVL and VLMEvalKit repositories for model evaluation' (DocVQA, ChartQA, TextVQA, AI2D via the InternVL repo; MMMU, OCRBench, RealWorldQA, HallBench, MathVista via VLMEvalKit); video: 16 frames per video at 448x448. OCRBench is reported out of 1000. Pinned card revision dated 2025-03-25.",
    "source_id": "card-opengvlab-internvl2-cards"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "mmmu",
    "score": 0.511,
    "source": "https://github.com/haotian-liu/LLaVA/blob/c121f0432da27facab705978f83c4ada465e46fd/docs/MODEL_ZOO.md",
    "by": "lab",
    "date": "2024-01-30",
    "note": "LLaVA authors' own results in the pinned LLaVA repository docs/MODEL_ZOO.md (Apache-2.0), 'LLaVA-v1.6' table, row 'LLaVA-1.6 | Hermes-Yi-34B' (checkpoint liuhaotian/llava-v1.6-34b), column 'MMMU' = 51.1. MMMU validation split per the LLaVA-NeXT release blog's 'MMMU (val)' column (same values). MathVista (split not stated) and VQAv2 (test-dev, not the catalogue's val id) not used.",
    "source_id": "llava-model-zoo-apache"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "textvqa",
    "score": 0.695,
    "source": "https://github.com/haotian-liu/LLaVA/blob/c121f0432da27facab705978f83c4ada465e46fd/docs/MODEL_ZOO.md",
    "by": "lab",
    "date": "2024-01-30",
    "note": "LLaVA authors' own results in the pinned LLaVA repository docs/MODEL_ZOO.md (Apache-2.0), 'LLaVA-v1.6' table, row 'LLaVA-1.6 | Hermes-Yi-34B' (checkpoint liuhaotian/llava-v1.6-34b), column 'TextVQA' = 69.5. TextVQA val (docs/Evaluation.md at the same commit evaluates TextVQA_0.5.1_val). MathVista (split not stated) and VQAv2 (test-dev, not the catalogue's val id) not used.",
    "source_id": "llava-model-zoo-apache"
  },
  {
    "model": "llava-v1-6-vicuna-13b",
    "bench": "mmmu",
    "score": 0.362,
    "source": "https://github.com/haotian-liu/LLaVA/blob/c121f0432da27facab705978f83c4ada465e46fd/docs/MODEL_ZOO.md",
    "by": "lab",
    "date": "2024-01-30",
    "note": "LLaVA authors' own results in the pinned LLaVA repository docs/MODEL_ZOO.md (Apache-2.0), 'LLaVA-v1.6' table, row 'LLaVA-1.6 | Vicuna-13B' (checkpoint liuhaotian/llava-v1.6-vicuna-13b), column 'MMMU' = 36.2. MMMU validation split per the LLaVA-NeXT release blog's 'MMMU (val)' column (same values). MathVista (split not stated) and VQAv2 (test-dev, not the catalogue's val id) not used.",
    "source_id": "llava-model-zoo-apache"
  },
  {
    "model": "llava-v1-6-vicuna-13b",
    "bench": "textvqa",
    "score": 0.671,
    "source": "https://github.com/haotian-liu/LLaVA/blob/c121f0432da27facab705978f83c4ada465e46fd/docs/MODEL_ZOO.md",
    "by": "lab",
    "date": "2024-01-30",
    "note": "LLaVA authors' own results in the pinned LLaVA repository docs/MODEL_ZOO.md (Apache-2.0), 'LLaVA-v1.6' table, row 'LLaVA-1.6 | Vicuna-13B' (checkpoint liuhaotian/llava-v1.6-vicuna-13b), column 'TextVQA' = 67.1. TextVQA val (docs/Evaluation.md at the same commit evaluates TextVQA_0.5.1_val). MathVista (split not stated) and VQAv2 (test-dev, not the catalogue's val id) not used.",
    "source_id": "llava-model-zoo-apache"
  },
  {
    "model": "cogvlm2-llama3-chat-19b",
    "bench": "textvqa",
    "score": 0.842,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3', column 'TextVQA' = 84.2 (divided by 100). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "cogvlm2-llama3-chat-19b",
    "bench": "chartqa",
    "score": 0.81,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3', column 'ChartQA' = 81.0 (divided by 100). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "cogvlm2-llama3-chat-19b",
    "bench": "ocrbench",
    "score": 0.756,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3', column 'OCRbench' = 756 (divided by 1000). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "cogvlm2-llama3-chinese-chat-19b",
    "bench": "textvqa",
    "score": 0.85,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3-Chinese', column 'TextVQA' = 85.0 (divided by 100). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "cogvlm2-llama3-chinese-chat-19b",
    "bench": "chartqa",
    "score": 0.747,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3-Chinese', column 'ChartQA' = 74.7 (divided by 100). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "cogvlm2-llama3-chinese-chat-19b",
    "bench": "ocrbench",
    "score": 0.78,
    "source": "https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B/blob/f592f291cf528389b2e4776b1e84ecdf6d71fbe3/README.md#benchmark",
    "by": "lab",
    "date": "2024-09-03",
    "note": "Zhipu AI / THUDM's own result in the pinned cogvlm2-llama3-chat-19B model card (zai-org, formerly THUDM), 'Benchmark' table, row 'CogVLM2-LLaMA3-Chinese', column 'OCRbench' = 780 (divided by 1000). Card: 'All reviews were obtained without using any external OCR tools (\"pixel only\")'. Splits not stated, so only benchmark ids without a fixed split are used; DocVQA (catalogue id is test), MMMU (id is val), VCR, MMVet and MMBench not used. Competitor rows (CogVLM1.1, LLaVA, InternVL-1.5, QwenVL-Plus, Claude3-Opus, Gemini Pro 1.5, GPT-4V) not used. Vendor-reported (owner rule).",
    "source_id": "first-party-zai-cogvlm2-scores"
  },
  {
    "model": "kolors",
    "bench": "hpsv3-benchmark",
    "score": 10.55,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'Kolors', column 'Overall' = 10.55 (raw HPSv3 score, higher is better; kept in raw units). Scored by the HPSv3 authors with their HPSv3 (Qwen2-VL-7B-based) preference model over the benchmark's 12 prompt categories; the README's HPDv3 data table lists each model's generated images among the authors' own collection (e.g. Kolors 49,705, SDXL 49,025, SD3 Medium 49,266, FLUX.1 dev 48,927 images, train & test). Model-based score, not human votes.",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "flux-1-dev",
    "bench": "hpsv3-benchmark",
    "score": 10.43,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'Flux-dev', column 'Overall' = 10.43 (raw HPSv3 score, higher is better; kept in raw units). Scored by the HPSv3 authors with their HPSv3 (Qwen2-VL-7B-based) preference model over the benchmark's 12 prompt categories; the README's HPDv3 data table lists each model's generated images among the authors' own collection (e.g. Kolors 49,705, SDXL 49,025, SD3 Medium 49,266, FLUX.1 dev 48,927 images, train & test). Model-based score, not human votes.",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "playground-v2-5",
    "bench": "hpsv3-benchmark",
    "score": 10.27,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'Playgroundv2.5', column 'Overall' = 10.27 (raw HPSv3 score, higher is better; kept in raw units). Scored by the HPSv3 authors with their HPSv3 (Qwen2-VL-7B-based) preference model over the benchmark's 12 prompt categories; the README's HPDv3 data table lists each model's generated images among the authors' own collection (e.g. Kolors 49,705, SDXL 49,025, SD3 Medium 49,266, FLUX.1 dev 48,927 images, train & test). Model-based score, not human votes.",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "sdxl",
    "bench": "hpsv3-benchmark",
    "score": 8.2,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'SDXL', column 'Overall' = 8.20 (raw HPSv3 score, higher is better; kept in raw units). Scored by the HPSv3 authors with their HPSv3 (Qwen2-VL-7B-based) preference model over the benchmark's 12 prompt categories; the README's HPDv3 data table lists each model's generated images among the authors' own collection (e.g. Kolors 49,705, SDXL 49,025, SD3 Medium 49,266, FLUX.1 dev 48,927 images, train & test). Model-based score, not human votes.",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "sd-3-medium",
    "bench": "hpsv3-benchmark",
    "score": 5.31,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'Stable Diffusion 3 Medium', column 'Overall' = 5.31 (raw HPSv3 score, higher is better; kept in raw units). Scored by the HPSv3 authors with their HPSv3 (Qwen2-VL-7B-based) preference model over the benchmark's 12 prompt categories; the README's HPDv3 data table lists each model's generated images among the authors' own collection (e.g. Kolors 49,705, SDXL 49,025, SD3 Medium 49,266, FLUX.1 dev 48,927 images, train & test). Model-based score, not human votes.",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "veo-3",
    "bench": "pai-bench-g-overall",
    "score": 0.822,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU) Table 3, row 'Veo3 [21]', column 'Overall' = 82.2 (stored /100): mean of the Domain Score (MLLM accuracy on per-case physical/semantic QA over six Physical-AI domains) and the Quality Score (VBench-style consistency, smoothness, aesthetic and imaging metrics) on PAI-Bench-G. Authors' own run: 'We evaluate all models using their default settings ... scores for open-source models are averaged over five videos generated per caption'. API model veo-3.0-generate-001 (Appendix Table 7), one video per caption.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "hunyuanvideo-i2v",
    "bench": "pai-bench-g-overall",
    "score": 0.776,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU) Table 3, row 'HunyuanVideo-I2V [43]', column 'Overall' = 77.6 (stored /100): mean of the Domain Score (MLLM accuracy on per-case physical/semantic QA over six Physical-AI domains) and the Quality Score (VBench-style consistency, smoothness, aesthetic and imaging metrics) on PAI-Bench-G. Authors' own run: 'We evaluate all models using their default settings ... scores for open-source models are averaged over five videos generated per caption'. default settings, 129 frames, 24 fps, 1184x768, five seeds per caption.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "pai-bench-g-overall",
    "score": 0.813,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU) Table 3, row 'Wan2.2-TI2V-5B [79]', column 'Overall' = 81.3 (stored /100): mean of the Domain Score (MLLM accuracy on per-case physical/semantic QA over six Physical-AI domains) and the Quality Score (VBench-style consistency, smoothness, aesthetic and imaging metrics) on PAI-Bench-G. Authors' own run: 'We evaluate all models using their default settings ... scores for open-source models are averaged over five videos generated per caption'. default settings, 121 frames, 24 fps, 1248x704, five seeds per caption.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "pai-bench-g-overall",
    "score": 0.823,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU) Table 3, row 'Wan2.2-I2V-A14B [79]', column 'Overall' = 82.3 (stored /100): mean of the Domain Score (MLLM accuracy on per-case physical/semantic QA over six Physical-AI domains) and the Quality Score (VBench-style consistency, smoothness, aesthetic and imaging metrics) on PAI-Bench-G. Authors' own run: 'We evaluate all models using their default settings ... scores for open-source models are averaged over five videos generated per caption'. default settings, 81 frames, 16 fps, 1280x720, five seeds per caption.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "veo-3",
    "bench": "pai-bench-g-human-elo",
    "score": 1112.2,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'Veo3 [21]', column 'ELO Scores / Overall' = 1112.2. Arena-style human study: participants compare video pairs generated from identical captions on video quality and physical plausibility (or tie); preferences aggregated into ELO. Same generated videos as the Table 3 row (API model veo-3.0-generate-001 (Appendix Table 7), one video per caption).",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "hunyuanvideo-i2v",
    "bench": "pai-bench-g-human-elo",
    "score": 964.3,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'HunyuanVideo-I2V [43]', column 'ELO Scores / Overall' = 964.3. Arena-style human study: participants compare video pairs generated from identical captions on video quality and physical plausibility (or tie); preferences aggregated into ELO. Same generated videos as the Table 3 row (default settings, 129 frames, 24 fps, 1184x768, five seeds per caption).",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "pai-bench-g-human-elo",
    "score": 1082.6,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'Wan2.2-I2V-A14B [79]', column 'ELO Scores / Overall' = 1082.6. Arena-style human study: participants compare video pairs generated from identical captions on video quality and physical plausibility (or tie); preferences aggregated into ELO. Same generated videos as the Table 3 row (default settings, 81 frames, 16 fps, 1280x720, five seeds per caption).",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "gpt-image-2-5-sunburst",
    "bench": "genexam-relaxed",
    "score": 0.953,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'GPT-Image-2.5-sunburst', column 'Overall' = 95.3 (stored /100; strict Overall 78.9, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "gpt-image-2-5-flare",
    "bench": "genexam-relaxed",
    "score": 0.946,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'GPT-Image-2.5-flare', column 'Overall' = 94.6 (stored /100; strict Overall 74.8, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "gpt-image-2",
    "bench": "genexam-relaxed",
    "score": 0.938,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'GPT-Image-2', column 'Overall' = 93.8 (stored /100; strict Overall 74.6, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "qwen-image-3-0-pro",
    "bench": "genexam-relaxed",
    "score": 0.937,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Qwen-Image-3.0-pro', column 'Overall' = 93.7 (stored /100; strict Overall 67.5, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run. Main row (with prompt enhancement, the API default); the '(w/o PE)' row is not used.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "nano-banana-2",
    "bench": "genexam-relaxed",
    "score": 0.926,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Nano Banana 2', column 'Overall' = 92.6 (stored /100; strict Overall 64.1, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "seedream-5-0-pro",
    "bench": "genexam-relaxed",
    "score": 0.9,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Seedream 5.0 Pro', column 'Overall' = 90.0 (stored /100; strict Overall 54.3, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "nano-banana",
    "bench": "genexam-relaxed",
    "score": 0.559,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Gemini-2.5-Flash-Image', column 'Overall' = 55.9 (stored /100; strict Overall 3.9, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run. 'Gemini-2.5-Flash-Image' = Nano Banana.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "flux-1-kontext-max",
    "bench": "genexam-relaxed",
    "score": 0.285,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'FLUX.1 Kontext max', column 'Overall' = 28.5 (stored /100; strict Overall 0.0, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "seedream-3-0",
    "bench": "genexam-relaxed",
    "score": 0.234,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Seedream 3.0', column 'Overall' = 23.4 (stored /100; strict Overall 0.2, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "qwen-image-2-1",
    "bench": "genexam-relaxed",
    "score": 0.471,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Qwen-Image-2.1', column 'Overall' = 47.1 (stored /100; strict Overall 5.3, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "qwen-image",
    "bench": "genexam-relaxed",
    "score": 0.275,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'Qwen-Image', column 'Overall' = 27.5 (stored /100; strict Overall 0.3, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "genexam-relaxed",
    "score": 0.191,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'HunyuanImage-3.0', column 'Overall' = 19.1 (stored /100; strict Overall 0.0, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "flux-1-dev",
    "bench": "genexam-relaxed",
    "score": 0.176,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'FLUX.1 dev', column 'Overall' = 17.6 (stored /100; strict Overall 0.0, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run.",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "flux-1-krea-dev",
    "bench": "genexam-relaxed",
    "score": 0.164,
    "source": "https://github.com/OpenGVLab/GenExam/blob/1f820dbdb3fa802c2ea893acbf5ce5ada9d766d1/README.md",
    "by": "independent",
    "date": "2026-09-26",
    "note": "GenExam live leaderboard in the OpenGVLab/GenExam README (@1f820db, MIT; news entry 2026-09-26), 'Relaxed Score' table, row 'FLUX.1 Krea', column 'Overall' = 16.4 (stored /100; strict Overall 0.0, not recorded). 1,000 exam-style T2I problems over 10 subjects with ground-truth images and scoring points; judged by gpt-5-20250807 (README evaluation instructions). Recorded only for models with no genexam-relaxed row from the paper (arXiv 2509.14232v5): the README re-scores several paper models with slightly different values, so it is kept as a separate run. 'FLUX.1 Krea' listed under open-source T2I models = FLUX.1 Krea [dev].",
    "source_id": "genexam-readme-mit"
  },
  {
    "model": "step1x-edit-v1-1",
    "bench": "gedit-bench-overall-lang-unspecified",
    "score": 0.697,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "by": "lab",
    "date": "2025-12-29",
    "note": "StepFun's own results in the pinned Step1X-Edit-v1p2 model card (Apache-2.0), first table, row 'Step1X-Edit v1.1' (StepFun's previous release, same table and protocol as the existing v1p2 rows), column 'GEdit-Bench G_O⬆️' = 6.97 (divided by 10). GEdit-Bench G_O on the 0-10 scale; language split and judge not stated in the card (same id as the v1p2 row). Competitor rows (Flux-Kontext-dev, Qwen-Image-Edit-2509) and the preview/base/thinking rows not used.",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2"
  },
  {
    "model": "step1x-edit-v1-1",
    "bench": "kris-bench-overall",
    "score": 0.5159,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "by": "lab",
    "date": "2025-12-29",
    "note": "StepFun's own results in the pinned Step1X-Edit-v1p2 model card (Apache-2.0), first table, row 'Step1X-Edit v1.1' (StepFun's previous release, same table and protocol as the existing v1p2 rows), column 'Kris-Bench Overall⬆️' = 51.59 (divided by 100). KRIS-Bench Overall converted from percent. Competitor rows (Flux-Kontext-dev, Qwen-Image-Edit-2509) and the preview/base/thinking rows not used.",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "caparena-auto-score",
    "score": 56.17,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'Gemini-1.5-pro-002', column 'Score_Avg' = 56.17 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "caparena-auto-score",
    "score": 44,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'GPT-4o-0806', column 'Score_Avg' = 44.00 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.' This model is also one of the three baselines (its own-baseline column is 0 by construction).",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "caparena-auto-score",
    "score": 35.33,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'Qwen2.5-VL-72B-Instruct', column 'Score_Avg' = 35.33 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "caparena-auto-score",
    "score": 21.5,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'Claude-3.5-Sonnet-0620', column 'Score_Avg' = 21.50 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "internvl2-26b",
    "bench": "caparena-auto-score",
    "score": 13,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'InternVL2-26B', column 'Score_Avg' = 13.00 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "caparena-auto-score",
    "score": 9.33,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'GPT-4o-mini-0718', column 'Score_Avg' = 9.33 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "cogvlm2-llama3-chat-19b",
    "bench": "caparena-auto-score",
    "score": -8.5,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'CogVLM2-llama3-chat-19B', column 'Score_Avg' = -8.50 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.' This model is also one of the three baselines (its own-baseline column is 0 by construction).",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "caparena-auto-score",
    "score": -9,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'Qwen2-VL-72B-Instruct', column 'Score_Avg' = -9.00 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "llava-onevision-qwen2-72b-ov",
    "bench": "caparena-auto-score",
    "score": -12.33,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'LLaVA-Onevision-72B-sft', column 'Score_Avg' = -12.33 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.' Label 'LLaVA-Onevision-72B-sft' = the ov-sft checkpoint (llava-onevision-qwen2-72b-ov).",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "caparena-auto-score",
    "score": -25.67,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'LLama-3.2-vision-90B-Instruct', column 'Score_Avg' = -25.67 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "minicpm-v-2-6",
    "bench": "caparena-auto-score",
    "score": -38,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'MiniCPM-V2.6-8B', column 'Score_Avg' = -38.00 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.' This model is also one of the three baselines (its own-baseline column is 0 by construction).",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "caparena-auto-score",
    "score": -49,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'Qwen2-VL-7B-Instruct', column 'Score_Avg' = -49.00 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "caparena-auto-score",
    "score": -67.5,
    "source": "https://arxiv.org/html/2503.12329v1",
    "by": "independent",
    "date": "2025-03-16",
    "note": "CapArena (arXiv 2503.12329v1, Nanjing University) Appendix J Table 8 'CapArena-Auto Leaderboard', row 'LLaVA-1.6-34B', column 'Score_Avg' = -67.50 (raw units, -100..100). CapArena-Auto: 600 DOCCI test images; each test model's detailed caption is compared with captions from three baselines (GPT-4o, CogVLM2-19B, MiniCPM-V2.6-8B) by a GPT-4o judge given the human reference caption; +1 win, -1 loss, 0 draw; the text defines the score as the sum over the 600 samples, and the table prints per-baseline values on a -100..100 scale (a baseline scores 0 against itself) with Score_Avg their mean (per-baseline columns not recorded). Authors' own run: 'We provide the current leaderboard of CapArena-Auto in Table 8.'",
    "source_id": "paper-2503.12329v1-caparena"
  },
  {
    "model": "step1x-edit-v1-2-preview",
    "bench": "gedit-bench-overall-lang-unspecified",
    "score": 0.742,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "by": "lab",
    "date": "2025-12-29",
    "note": "StepFun's own results in the pinned Step1X-Edit-v1p2 model card (Apache-2.0), first table, row 'Step1x-edit-v1p2-preview' (the separately released preview checkpoint, default mode), column 'GEdit-Bench G_O⬆️' = 7.42 (divided by 10). Same table and protocol as the v1.1 and v1p2 rows.",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2"
  },
  {
    "model": "step1x-edit-v1-2-preview",
    "bench": "kris-bench-overall",
    "score": 0.5251,
    "source": "https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2/blob/ca85b97fd19f2235dc0d6fd3633d1319f169e149/README.md",
    "by": "lab",
    "date": "2025-12-29",
    "note": "StepFun's own results in the pinned Step1X-Edit-v1p2 model card (Apache-2.0), first table, row 'Step1x-edit-v1p2-preview' (the separately released preview checkpoint, default mode), column 'Kris-Bench Overall⬆️' = 52.51 (divided by 100). Same table and protocol as the v1.1 and v1p2 rows.",
    "source_id": "card2-stepfun-ai-step1x-edit-v1p2"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "physbench-test-avg",
    "score": 0.37,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Table 3, row 'Claude-3-opus (Anthropic, 2024)', column 'Avg' = 37.00 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "physbench-test-avg",
    "score": 0.3618,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Table 3, row 'Claude-3-sonnet (Anthropic, 2024)', column 'Avg' = 36.18 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "physbench-test-avg",
    "score": 0.3944,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Table 3, row 'Claude-3-haiku (Anthropic, 2024)', column 'Avg' = 39.44 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "physbench-test-avg",
    "score": 0.4315,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Table 3, row 'GPT-4o-mini (Achiam et al., 2023)', column 'Avg' = 43.15 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'seq'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11. Only one GPT-4o mini snapshot exists (2024-07-18).",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "physbench-test-avg",
    "score": 0.3842,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Table 3, row 'Phi-3V (Abdin et al., 2024)', column 'Avg' = 38.42 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'seq'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11. 'Phi-3V' (4B) = Phi-3-vision-128k-instruct, the only Phi-3 vision release.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "physbench-test-avg",
    "score": 0.3975,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Appendix Table 28 (latest 36 VLMs), row 'Phi-3.5V (AzureML, 2024)', column 'Avg' = 39.75 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'seq'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "internvl2-4b",
    "bench": "physbench-test-avg",
    "score": 0.3971,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Appendix Table 28 (latest 36 VLMs), row 'InternVL2-4B (Wang et al., 2024e)', column 'Avg' = 39.71 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'seq'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "internvl2-26b",
    "bench": "physbench-test-avg",
    "score": 0.435,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Appendix Table 28 (latest 36 VLMs), row 'InternVL2-26B (Wang et al., 2024e)', column 'Avg' = 43.50 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11.",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "minicpm-v-2-6",
    "bench": "physbench-test-avg",
    "score": 0.4489,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Appendix Table 28 (latest 36 VLMs), row 'MiniCPM2.6 (Yao et al., 2024)', column 'Avg' = 44.89 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11. 'MiniCPM2.6' (8B) = MiniCPM-V 2.6 (generation settings listed as MiniCPM-V2.6 in Appendix Table 9).",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "physbench-test-avg",
    "score": 0.5511,
    "source": "https://arxiv.org/html/2501.16411v2",
    "by": "independent",
    "date": "2025-01-29",
    "note": "PhysBench (arXiv 2501.16411v2, ICLR 2025) Appendix Table 28 (latest 36 VLMs), row 'o1 (Jaech et al., 2024)', column 'Avg' = 55.11 (stored /100). Test set of 10,002 interleaved video-image-text multiple-choice questions over physical object property, relationships, scene understanding and dynamics (4 options, random choice 25.00); format 'merge'. Authors' own run: 'we evaluated 75 representative VLMs on PhysBench' with per-model generation settings in Appendix Tables 9-11. 'o1' (not o1-preview/o1-mini) evaluated for the Jan 2025 revision; mapped to the o1 GA snapshot 2024-12-17 (review flag).",
    "source_id": "paper-2501.16411v2-physbench"
  },
  {
    "model": "llava-onevision-qwen2-72b-ov",
    "bench": "msearth-mcq-overall",
    "score": 0.5111,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'LLaVA-onvision-72B', column 'Overall ACC' = 51.11 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Appendix Table 10 gives the checkpoint path llava-hf/llava-onevision-qwen2-72b-ov-hf (the ov-sft checkpoint, HF-format).",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "msearth-mcq-overall",
    "score": 0.5065,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'Qwen2.5-VL-72B', column 'Overall ACC' = 50.65 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Checkpoint Qwen/Qwen2.5-VL-72B-Instruct (Appendix Table 10).",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "msearth-mcq-overall",
    "score": 0.6128,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'Gemini-2.5-Pro-Thinking', column 'Overall ACC' = 61.28 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Version gemini-2.5-pro-preview-05-06 (Appendix Table 10), thinking on.",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "msearth-mcq-overall",
    "score": 0.5722,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'Gemini-2.5-Flash-Thinking', column 'Overall ACC' = 57.22 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Version Gemini-2.5-Flash-preview-04-17 (Appendix Table 10), thinking row used (highest reasoning setting; the non-thinking row prints 56.11).",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "msearth-mcq-overall",
    "score": 0.5801,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'Claude-3.7-Sonnet', column 'Overall ACC' = 58.01 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Version Claude-3.7-Sonnet-20250219 (Appendix Table 10).",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "msearth-mcq-overall",
    "score": 0.4759,
    "source": "https://arxiv.org/html/2505.20740v3",
    "by": "independent",
    "date": "2026-05-03",
    "note": "MSEarth (arXiv 2505.20740v3, v3 3 May 2026) Table 4 'Accuracies (%) of different models on multiple-choice questions', row 'Claude-3.5-Haiku', column 'Overall ACC' = 47.59 (stored /100). MSEarth-Bench MCQ test set: 2,784 graduate-level Earth-science figure questions with four options (A-D), answer extracted by regular expression. Authors' own run (Sec. 4.1 'We evaluate different families of MLLMs on our benchmark ... We use chat/instruction-tuned variants of each model'). Not in the version table; Claude 3.5 Haiku has a single snapshot (20241022).",
    "source_id": "paper-2505.20740v3-msearth"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "mmlu",
    "score": 0.557,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'MMLU' = 55.7 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "mmlu",
    "score": 0.497,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'MMLU' = 49.7 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "bbh",
    "score": 0.481,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'BBH' = 48.1 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "bbh",
    "score": 0.431,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'BBH' = 43.1 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "c-eval",
    "score": 0.601,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'C-Eval' = 60.1 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "c-eval",
    "score": 0.447,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'C-Eval' = 44.7 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "cmmlu",
    "score": 0.625,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'CMMLU' = 62.5 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "cmmlu",
    "score": 0.512,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'CMMLU' = 51.2 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "humaneval",
    "score": 0.573,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'HumanEval' = 57.3 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "humaneval",
    "score": 0.451,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'HumanEval' = 45.1 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "mbpp",
    "score": 0.458,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'MBPP' = 45.8 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "mbpp",
    "score": 0.39,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'MBPP' = 39.0 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "gsm8k",
    "score": 0.72,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'GSM8K' = 72.0 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "gsm8k",
    "score": 0.626,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'GSM8K' = 62.6 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "math",
    "score": 0.279,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek-V2-Lite 16B Chat (SFT)', row 'Math' = 27.9 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "deepseek-llm-7b-chat",
    "bench": "math",
    "score": 0.147,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/85864749cd611b4353ce1decdb286193298f64c7/README.md#chat-model",
    "by": "lab",
    "date": "2024-06-25",
    "note": "DeepSeek's own result in the pinned DeepSeek-V2-Lite-Chat model card, '4. Evaluation Results / Chat Model / Standard Benchmark' table, column 'DeepSeek 7B Chat (SFT)', row 'Math' = 14.7 (stored /100). The card defers few-shot settings and prompts to the DeepSeek-V2 paper ('For more evaluation details, such as few-shot settings and prompts, please check our paper'). Both columns are DeepSeek's own models; the DeepSeekMoE 16B Chat column is also own but not recorded (one partner suffices). Vendor-reported (owner rule).",
    "source_id": "first-party-deepseek-scores"
  },
  {
    "model": "imagen-3",
    "bench": "geneval",
    "score": 0.707,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Imagen 3', column 'GenEval' = 0.707 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'imagen-3.0-generate-002' (Gemini API) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "recraft-v3",
    "bench": "geneval",
    "score": 0.732,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Recraft v3', column 'GenEval' = 0.732 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'recraftv3' (Recraft API) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "luma-photon",
    "bench": "geneval",
    "score": 0.738,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Luma Photon', column 'GenEval' = 0.738 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'luma/photon' (Replicate) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "flux-1-1-pro",
    "bench": "geneval",
    "score": 0.588,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Flux 1.1 Pro', column 'GenEval' = 0.588 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'black-forest-labs/flux-1.1-pro' (Replicate) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "ideogram-2",
    "bench": "geneval",
    "score": 0.615,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Ideogram 2', column 'GenEval' = 0.615 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'ideogram-ai/ideogram-v2' (Replicate) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "dall-e-3",
    "bench": "geneval",
    "score": 0.627,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Dalle 3', column 'GenEval' = 0.627 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'dall-e-3' (OpenAI API) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "sd-3-5-large",
    "bench": "geneval",
    "score": 0.591,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'SD 3.5', column 'GenEval' = 0.591 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'stabilityai/stable-diffusion-3.5-large' (Hugging Face) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "gemini-2-0-flash-image",
    "bench": "geneval",
    "score": 0.669,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'Gemini 2 Image', column 'GenEval' = 0.669 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'gemini-2.0-flash-preview-image-generation' (Gemini API) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "gpt-image-1",
    "bench": "geneval",
    "score": 0.808,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Table 3, row 'GPT Image', column 'GenEval' = 0.808 (overall, 0-1). The MMMG authors ran GenEval themselves on the nine image models for their correlation study (Sec. 4.3: 'scores produced by evaluating on different benchmarks in Table 3. We compare with GenEval, DrawBench, and GenAI-Bench'); checkpoint 'gpt-image-1' (OpenAI API) per Appendix C. The Arena column (copied from Chatbot Arena) and the DrawBench/GenAI-Bench VQAScore columns (GenAI-Bench on a random 400-prompt subsample) are not used.",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "imagen-3",
    "bench": "mmmg-image-avg",
    "score": 0.474,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Imagen 3', row 'Average' = 0.474 (0-1; equals the MMMG column of Table 3). Checkpoint 'imagen-3.0-generate-002' (Gemini API) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "recraft-v3",
    "bench": "mmmg-image-avg",
    "score": 0.441,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Recraft v3', row 'Average' = 0.441 (0-1; equals the MMMG column of Table 3). Checkpoint 'recraftv3' (Recraft API) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "luma-photon",
    "bench": "mmmg-image-avg",
    "score": 0.587,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Luma Photon', row 'Average' = 0.587 (0-1; equals the MMMG column of Table 3). Checkpoint 'luma/photon' (Replicate) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "flux-1-1-pro",
    "bench": "mmmg-image-avg",
    "score": 0.431,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Flux 1.1 Pro', row 'Average' = 0.431 (0-1; equals the MMMG column of Table 3). Checkpoint 'black-forest-labs/flux-1.1-pro' (Replicate) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "ideogram-2",
    "bench": "mmmg-image-avg",
    "score": 0.508,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Ideo -gram 2', row 'Average' = 0.508 (0-1; equals the MMMG column of Table 3). Checkpoint 'ideogram-ai/ideogram-v2' (Replicate) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "dall-e-3",
    "bench": "mmmg-image-avg",
    "score": 0.352,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Dalle 3', row 'Average' = 0.352 (0-1; equals the MMMG column of Table 3). Checkpoint 'dall-e-3' (OpenAI API) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "sd-3-5-large",
    "bench": "mmmg-image-avg",
    "score": 0.332,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'SD 3.5', row 'Average' = 0.332 (0-1; equals the MMMG column of Table 3). Checkpoint 'stabilityai/stable-diffusion-3.5-large' (Hugging Face) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "gemini-2-0-flash-image",
    "bench": "mmmg-image-avg",
    "score": 0.592,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Gemini 2 Image', row 'Average' = 0.592 (0-1; equals the MMMG column of Table 3). Checkpoint 'gemini-2.0-flash-preview-image-generation' (Gemini API) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "gpt-image-1",
    "bench": "mmmg-image-avg",
    "score": 0.707,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'GPT Image', row 'Average' = 0.707 (0-1; equals the MMMG column of Table 3). Checkpoint 'gpt-image-1' (OpenAI API) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "nano-banana",
    "bench": "mmmg-image-avg",
    "score": 0.654,
    "source": "https://arxiv.org/html/2505.17613v2",
    "by": "independent",
    "date": "2026-09-30",
    "note": "MMMG (arXiv 2505.17613v2) Appendix Table 9 'Benchmarking results of ... models on 15 image generation tasks', column 'Gemini 2.5 Image', row 'Average' = 0.654 (0-1; equals the MMMG column of Table 3). Checkpoint 'gemini-2.5-flash-image-preview' (Gemini API), the preview endpoint of Gemini 2.5 Flash Image (Nano Banana) per Appendix C (Evaluated Models). Each task is scored with the evaluation method the authors found most aligned with human judgment (VLM judge or program).",
    "source_id": "paper-2505.17613v2-mmmg"
  },
  {
    "model": "emu3",
    "bench": "mmmg-kg-score-avg",
    "score": 0.0611,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'Emu-3', column 'Avg' = 6.11 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "janus-pro-7b",
    "bench": "mmmg-kg-score-avg",
    "score": 0.131,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'Janus-pro-7B', column 'Avg' = 13.10 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "ideogram-2",
    "bench": "mmmg-kg-score-avg",
    "score": 0.1221,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'Ideogram', column 'Avg' = 12.21 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly. Table label 'Ideogram'; the text lists the evaluated model as 'Ideogram 2.0'.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "bagel",
    "bench": "mmmg-kg-score-avg",
    "score": 0.1502,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'BAGEL', column 'Avg' = 15.02 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "sdxl",
    "bench": "mmmg-kg-score-avg",
    "score": 0.159,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'SDXL-1.0', column 'Avg' = 15.90 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly. SDXL-1.0 base row, not '-refiner'.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "flux-1-dev",
    "bench": "mmmg-kg-score-avg",
    "score": 0.1913,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'FLUX.1-[dev]', column 'Avg' = 19.13 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly. Plain FLUX.1-[dev] row, not the '(recaption)' variant.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "hidream-i1-full",
    "bench": "mmmg-kg-score-avg",
    "score": 0.2804,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (Massive Multi-discipline Multi-tier Generation, arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'HiDream-I1-Full', column 'Avg' = 28.04 (MMMG-Score x100, stored /100): mean over six education levels (preschool to PhD) of knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph OpenAI-o3 grounds in the image) multiplied by a segment-count readability score; 4,456 prompts. Authors' own run: 'comprehensive evaluations of state-of-the-art text-to-image models ... on the MMMG benchmark'; open models at default settings (diffusion models with CFG 3.5), API models queried directly.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "dall-e-3",
    "bench": "real-realism-avg",
    "score": 0.5244,
    "source": "https://arxiv.org/html/2502.10663v1",
    "by": "independent",
    "date": "2025-02-15",
    "note": "REAL (arXiv 2502.10663v1) Table 5 'Realism benchmark for four popular T2I models', row 'DALL-E 3', column 'Average Score' = 0.5244 (0-1): mean of the attribute, relationship and style realism scores computed by REAL's VQA-based schema on generated images for iNaturalist and CUB-Birds classes (attributes, style) and UnRel relationships. Authors' own run (Sec. 4.2 'we utilize four T2I models'; Sec. 4.5 'we propose using its output score to benchmark current T2I models').",
    "source_id": "paper-2502.10663v1-real"
  },
  {
    "model": "sd-1-1",
    "bench": "real-realism-avg",
    "score": 0.5271,
    "source": "https://arxiv.org/html/2502.10663v1",
    "by": "independent",
    "date": "2025-02-15",
    "note": "REAL (arXiv 2502.10663v1) Table 5 'Realism benchmark for four popular T2I models', row 'SD v1.1', column 'Average Score' = 0.5271 (0-1): mean of the attribute, relationship and style realism scores computed by REAL's VQA-based schema on generated images for iNaturalist and CUB-Birds classes (attributes, style) and UnRel relationships. Authors' own run (Sec. 4.2 'we utilize four T2I models'; Sec. 4.5 'we propose using its output score to benchmark current T2I models'). 'Stable Diffusion 1.1' (Sec. 4.2).",
    "source_id": "paper-2502.10663v1-real"
  },
  {
    "model": "kandinsky-3",
    "bench": "real-realism-avg",
    "score": 0.6324,
    "source": "https://arxiv.org/html/2502.10663v1",
    "by": "independent",
    "date": "2025-02-15",
    "note": "REAL (arXiv 2502.10663v1) Table 5 'Realism benchmark for four popular T2I models', row 'Kandinsky 3', column 'Average Score' = 0.6324 (0-1): mean of the attribute, relationship and style realism scores computed by REAL's VQA-based schema on generated images for iNaturalist and CUB-Birds classes (attributes, style) and UnRel relationships. Authors' own run (Sec. 4.2 'we utilize four T2I models'; Sec. 4.5 'we propose using its output score to benchmark current T2I models'). Kandinsky 3 (Arkhipkin et al. 2024), not 3.1.",
    "source_id": "paper-2502.10663v1-real"
  },
  {
    "model": "gemma-2-9b-simpo",
    "bench": "alpacaeval2-lc",
    "score": 0.7235,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gemma-2-9b-it-SimPO' (mode 'community'), length_controlled_winrate = 72.35 (LC standard error 0.52); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 65.86; average length 1833.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "alpacaeval2-lc",
    "score": 0.5746,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-4o-2024-05-13' (mode 'minimal'), length_controlled_winrate = 57.46; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 51.33; average length 1873.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "alpacaeval2-lc",
    "score": 0.5502,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-4-turbo-2024-04-09' (mode 'minimal'), length_controlled_winrate = 55.02; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 46.12; average length 1802.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "alpacaeval2-lc",
    "score": 0.5237,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-3-5-sonnet-20240620' (mode 'community'), length_controlled_winrate = 52.37; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 40.56; average length 1488.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "alpacaeval2-lc",
    "score": 0.5073,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-4o-mini-2024-07-18' (mode 'minimal'), length_controlled_winrate = 50.73 (LC standard error 0.83); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 44.65; average length 1861.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "alpacaeval2-lc",
    "score": 0.5,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt4_1106_preview' (mode 'minimal'), length_controlled_winrate = 50.00; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 50.00; average length 2049.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.4391,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen1.5-110B-Chat' (mode 'community'), length_controlled_winrate = 43.91; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 33.78; average length 1631.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "alpacaeval2-lc",
    "score": 0.4051,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-3-opus-20240229' (mode 'minimal'), length_controlled_winrate = 40.51; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 29.11; average length 1388.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen2-72b",
    "bench": "alpacaeval2-lc",
    "score": 0.3807,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen2-72B-Instruct' (mode 'verified'), length_controlled_winrate = 38.07 (LC standard error 0.90); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 29.85; average length 1626.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.3657,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen1.5-72B-Chat' (mode 'verified'), length_controlled_winrate = 36.57; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 26.50; average length 1549.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-0314",
    "bench": "alpacaeval2-lc",
    "score": 0.3531,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt4_0314' (mode 'verified'), length_controlled_winrate = 35.31; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 22.07; average length 1371.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "alpacaeval2-lc",
    "score": 0.3487,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-3-sonnet-20240229' (mode 'minimal'), length_controlled_winrate = 34.87; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 25.56; average length 1420.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-3-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.3442,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Meta-Llama-3-70B-Instruct' (mode 'minimal'), length_controlled_winrate = 34.42; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 33.18; average length 1919.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-large-2402",
    "bench": "alpacaeval2-lc",
    "score": 0.3265,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'mistral-large-2402' (mode 'verified'), length_controlled_winrate = 32.65; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 21.44; average length 1362.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "alpacaeval2-lc",
    "score": 0.3088,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Mixtral-8x22B-Instruct-v0.1' (mode 'verified'), length_controlled_winrate = 30.88; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 22.21; average length 1445.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-0613",
    "bench": "alpacaeval2-lc",
    "score": 0.3018,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt4_0613' (mode 'verified'), length_controlled_winrate = 30.18; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.76; average length 1140.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-medium-2312",
    "bench": "alpacaeval2-lc",
    "score": 0.2861,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'mistral-medium' (mode 'verified'), length_controlled_winrate = 28.61; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 21.86; average length 1500. Entry config model_name: mistral-medium (the API alias of mistral-medium-2312 at the time).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-2-0",
    "bench": "alpacaeval2-lc",
    "score": 0.2816,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-2' (mode 'verified'), length_controlled_winrate = 28.16; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 17.19; average length 1069. Entry config model_name: claude-2 (the API alias of Claude 2.0).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "yi-34b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.2719,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Yi-34B-Chat' (mode 'verified'), length_controlled_winrate = 27.19; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 29.66; average length 2123.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "dbrx-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.2538,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'dbrx-instruct' (mode 'verified'), length_controlled_winrate = 25.38; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 18.45; average length 1450.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-2-1",
    "bench": "alpacaeval2-lc",
    "score": 0.2525,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-2.1' (mode 'verified'), length_controlled_winrate = 25.25; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.73; average length 1096.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "alpacaeval2-lc",
    "score": 0.2465,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'xwinlm-70b-v0.1' (mode 'community'), length_controlled_winrate = 24.65; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 21.81; average length 1775.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.239,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen1.5-14B-Chat' (mode 'verified'), length_controlled_winrate = 23.90; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 18.65; average length 1607.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "alpacaeval2-lc",
    "score": 0.2369,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Mixtral-8x7B-Instruct-v0.1' (mode 'minimal'), length_controlled_winrate = 23.69; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 18.26; average length 1465.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "alpacaeval2-lc",
    "score": 0.2292,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Meta-Llama-3-8B-Instruct' (mode 'minimal'), length_controlled_winrate = 22.92 (LC standard error 0.85); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 22.57; average length 1899.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "alpacaeval2-lc",
    "score": 0.2235,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-3.5-turbo-0613' (mode 'community'), length_controlled_winrate = 22.35; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 14.10; average length 1331.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.2124,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-70b' (mode 'verified'), length_controlled_winrate = 21.24; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.98; average length 1418.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "alpacaeval2-lc",
    "score": 0.2061,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Mistral-7B-Instruct-v0.3' (mode 'verified'), length_controlled_winrate = 20.61 (LC standard error 0.78); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 16.69; average length 1581.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "alpacaeval2-lc",
    "score": 0.193,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-3.5-turbo-1106' (mode 'verified'), length_controlled_winrate = 19.30; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 9.18; average length 796.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "alpacaeval2-lc",
    "score": 0.1809,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-3.5-turbo-0301' (mode 'verified'), length_controlled_winrate = 18.09; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 9.62; average length 827.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.1784,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'deepseek-llm-67b-chat' (mode 'community'), length_controlled_winrate = 17.84; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 12.09; average length 1151.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "alpacaeval2-lc",
    "score": 0.1758,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'wizardlm-70b' (mode 'community'), length_controlled_winrate = 17.58; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 14.38; average length 1545. Entry config model_name: WizardLM/WizardLM-70B-V1.0.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "alpacaeval2-lc",
    "score": 0.1757,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'vicuna-33b-v1.3' (mode 'verified'), length_controlled_winrate = 17.57; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 12.71; average length 1479.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "alpacaeval2-lc",
    "score": 0.1711,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Mistral-7B-Instruct-v0.2' (mode 'minimal'), length_controlled_winrate = 17.11; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 14.72; average length 1676.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.1625,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'OpenHermes-2.5-Mistral-7B' (mode 'verified'), length_controlled_winrate = 16.25; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 10.34; average length 1107.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.1475,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen1.5-7B-Chat' (mode 'verified'), length_controlled_winrate = 14.75; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 11.77; average length 1594.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "alpacaeval2-lc",
    "score": 0.1469,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Starling-LM-7B-alpha' (mode 'community'), length_controlled_winrate = 14.69; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 14.25; average length 1895.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.1469,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'llama-2-70b-chat-hf' (mode 'verified'), length_controlled_winrate = 14.69; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 13.89; average length 1790.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "alpacaeval2-lc",
    "score": 0.1446,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'wizardlm-13b-v1.2' (mode 'community'), length_controlled_winrate = 14.46; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 12.03; average length 1635.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "alpacaeval2-lc",
    "score": 0.132,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'zephyr-7b-beta' (mode 'community'), length_controlled_winrate = 13.20; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 10.99; average length 1444.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "dolphin-2-2-1-mistral-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.1312,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'dolphin-2.2.1-mistral-7b' (mode 'community'), length_controlled_winrate = 13.12; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 9.04; average length 1130.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen-14b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.1238,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen-14B-Chat' (mode 'community'), length_controlled_winrate = 12.38; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 7.50; average length 1013.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "alpacaeval2-lc",
    "score": 0.1155,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-13b' (mode 'community'), length_controlled_winrate = 11.55; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 10.12; average length 1614.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-13b-v1-5",
    "bench": "alpacaeval2-lc",
    "score": 0.1048,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'vicuna-13b-v1.5' (mode 'community'), length_controlled_winrate = 10.48; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 6.72; average length 1061.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gemma-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.1043,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gemma-7b-it' (mode 'verified'), length_controlled_winrate = 10.43; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 6.94; average length 1115.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "alpacaeval2-lc",
    "score": 0.1029,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'zephyr-7b-alpha' (mode 'community'), length_controlled_winrate = 10.29; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 8.35; average length 1302.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.092,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-7b' (mode 'community'), length_controlled_winrate = 9.20; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 8.20; average length 1663.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "text-davinci-001",
    "bench": "alpacaeval2-lc",
    "score": 0.0903,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'text_davinci_001' (mode 'verified'), length_controlled_winrate = 9.03; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 2.76; average length 296.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.0844,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'llama-2-13b-chat-hf' (mode 'verified'), length_controlled_winrate = 8.44; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 7.70; average length 1513.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "alpacaeval2-lc",
    "score": 0.0762,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'vicuna-7b-v1.5' (mode 'community'), length_controlled_winrate = 7.62; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 4.80; average length 1083.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "guanaco-33b",
    "bench": "alpacaeval2-lc",
    "score": 0.0569,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'guanaco-33b' (mode 'verified'), length_controlled_winrate = 5.69; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 5.00; average length 1311.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gemma-2b",
    "bench": "alpacaeval2-lc",
    "score": 0.0544,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gemma-2b-it' (mode 'verified'), length_controlled_winrate = 5.44; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 3.40; average length 1041.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.0535,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'llama-2-7b-chat-hf' (mode 'verified'), length_controlled_winrate = 5.35; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 4.96; average length 1479.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "phi-2",
    "bench": "alpacaeval2-lc",
    "score": 0.044,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'phi-2' (mode 'community'), length_controlled_winrate = 4.40; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 2.35; average length 626.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "chatglm2-6b",
    "bench": "alpacaeval2-lc",
    "score": 0.0436,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'chatglm2-6b' (mode 'community'), length_controlled_winrate = 4.36; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 2.76; average length 1027.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "alpacaeval2-lc",
    "score": 0.0327,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'oasst-sft-pythia-12b' (mode 'verified'), length_controlled_winrate = 3.27; stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 1.79; average length 726. Entry config model_name: OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "alpacaeval1-winrate",
    "score": 0.977,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt4_1106_preview' (mode 'minimal'), win_rate = 97.70 (standard error 0.51); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 2049.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-medium-2312",
    "bench": "alpacaeval1-winrate",
    "score": 0.9683,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'mistral-medium' (mode 'minimal'), win_rate = 96.83 (standard error 0.61); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1500. Entry config model_name: mistral-medium (the API alias of mistral-medium-2312 at the time).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "xwin-lm-70b-v0-1",
    "bench": "alpacaeval1-winrate",
    "score": 0.9557,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'xwinlm-70b-v0.1' (mode 'community'), win_rate = 95.57 (standard error 0.72); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1775.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "alpacaeval1-winrate",
    "score": 0.9503,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'tulu-2-dpo-70b' (mode 'community'), win_rate = 95.03 (standard error 0.76); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1418.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "alpacaeval1-winrate",
    "score": 0.9478,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'Mixtral-8x7B-Instruct-v0.1' (mode 'minimal'), win_rate = 94.78 (standard error 0.78); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1465.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-0314",
    "bench": "alpacaeval1-winrate",
    "score": 0.9478,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt4_0314' (mode 'verified'), win_rate = 94.78 (standard error 0.75); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1371.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "yi-34b-chat",
    "bench": "alpacaeval1-winrate",
    "score": 0.9408,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'Yi-34B-Chat' (mode 'verified'), win_rate = 94.08 (standard error 0.83); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 2123.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-4-0613",
    "bench": "alpacaeval1-winrate",
    "score": 0.9378,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt4_0613' (mode 'verified'), win_rate = 93.78 (standard error 0.83); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1140.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "alpacaeval1-winrate",
    "score": 0.9278,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'Mistral-7B-Instruct-v0.2' (mode 'minimal'), win_rate = 92.78 (standard error 0.91); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1676.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "alpacaeval1-winrate",
    "score": 0.9266,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'llama-2-70b-chat-hf' (mode 'minimal'), win_rate = 92.66 (standard error 0.91); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1790.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-2-0",
    "bench": "alpacaeval1-winrate",
    "score": 0.9136,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'claude-2' (mode 'minimal'), win_rate = 91.36 (standard error 0.99); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1069. Entry config model_name: claude-2 (the API alias of Claude 2.0).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "alpacaeval1-winrate",
    "score": 0.906,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'zephyr-7b-beta' (mode 'community'), win_rate = 90.60 (standard error 1.03); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1444.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "alpacaeval1-winrate",
    "score": 0.8937,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt-3.5-turbo-0301' (mode 'verified'), win_rate = 89.37 (standard error 1.08); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 827.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "alpacaeval1-winrate",
    "score": 0.8917,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'wizardlm-13b-v1.2' (mode 'community'), win_rate = 89.17 (standard error 1.09); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1635.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "alpacaeval1-winrate",
    "score": 0.8899,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'vicuna-33b-v1.3' (mode 'verified'), win_rate = 88.99 (standard error 1.10); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1479.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "alpacaeval1-winrate",
    "score": 0.8812,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'tulu-2-dpo-13b' (mode 'community'), win_rate = 88.12 (standard error 1.14); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1614.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-2-1",
    "bench": "alpacaeval1-winrate",
    "score": 0.8708,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'claude-2.1' (mode 'minimal'), win_rate = 87.08 (standard error 1.18); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1096.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "alpacaeval1-winrate",
    "score": 0.8626,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt-3.5-turbo-1106' (mode 'verified'), win_rate = 86.26 (standard error 1.21); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 796.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "alpacaeval1-winrate",
    "score": 0.8576,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'zephyr-7b-alpha' (mode 'community'), win_rate = 85.76 (standard error 1.23); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1302.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "alpacaeval1-winrate",
    "score": 0.8422,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'tulu-2-dpo-7b' (mode 'community'), win_rate = 84.22 (standard error 1.29); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1663.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "alpacaeval1-winrate",
    "score": 0.8109,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'llama-2-13b-chat-hf' (mode 'minimal'), win_rate = 81.09 (standard error 1.38); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1513.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "alpacaeval1-winrate",
    "score": 0.7137,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'llama-2-7b-chat-hf' (mode 'minimal'), win_rate = 71.37 (standard error 1.59); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1479.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "guanaco-33b",
    "bench": "alpacaeval1-winrate",
    "score": 0.6596,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'guanaco-33b' (mode 'verified'), win_rate = 65.96 (standard error 1.67); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1311.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "text-davinci-003",
    "bench": "alpacaeval1-winrate",
    "score": 0.5,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'text_davinci_003' (mode 'minimal'), win_rate = 50.00 (standard error 0.00); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 307.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "chatglm2-6b",
    "bench": "alpacaeval1-winrate",
    "score": 0.4713,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'chatglm2-6b' (mode 'community'), win_rate = 47.13 (standard error 1.76); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1027.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "phi-2",
    "bench": "alpacaeval1-winrate",
    "score": 0.3066,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'phi-2' (mode 'community'), win_rate = 30.66 (standard error 1.61); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 626.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "alpacaeval1-winrate",
    "score": 0.2596,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'oasst-sft-pythia-12b' (mode 'verified'), win_rate = 25.96 (standard error 1.53); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 726. Entry config model_name: OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "text-davinci-001",
    "bench": "alpacaeval1-winrate",
    "score": 0.1517,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'text_davinci_001' (mode 'minimal'), win_rate = 15.17 (standard error 1.24); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 296.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "kling-2-5-turbo-pro",
    "bench": "lmarena-video",
    "score": 1219.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/ff77a170d5ec578fe4eb8c91e0084a6b1a20beac/text_to_video/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-22",
    "note": "text_to_video, category overall, split latest; exact entry 'kling-2.5-turbo-1080p'; 2,111 votes; published 95% interval 1202.4–1237.2. Kling 2.5 Turbo is sold in Standard (720p) and Pro (1080p) modes, so the 1080p entry is the Pro tier this catalogue model names.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "kling-2-5-turbo-pro",
    "bench": "lmarena-video-i2v",
    "score": 1275.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/ff77a170d5ec578fe4eb8c91e0084a6b1a20beac/image_to_video/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-22",
    "note": "image_to_video, category overall, split latest; exact entry 'kling-2.5-turbo-1080p'; 3,795 votes; published 95% interval 1263.0–1288.1. Kling 2.5 Turbo is sold in Standard (720p) and Pro (1080p) modes, so the 1080p entry is the Pro tier this catalogue model names.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.969,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.4-nano-2026-03-17': factual consistency rate 96.9% (hallucination rate 3.1%), answer rate 100.0%, average summary length 144.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.967,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-2.5-flash-lite': factual consistency rate 96.7% (hallucination rate 3.3%), answer rate 99.5%, average summary length 95.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-4",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'microsoft/Phi-4': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 80.7%, average summary length 120.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "snowflake-arctic-instruct",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.957,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'snowflake/snowflake-arctic-instruct': factual consistency rate 95.7% (hallucination rate 4.3%), answer rate 62.7%, average summary length 81.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-12b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.956,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemma-3-12b-it': factual consistency rate 95.6% (hallucination rate 4.4%), answer rate 97.4%, average summary length 89.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-8b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.952,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-8b': factual consistency rate 95.2% (hallucination rate 4.8%), answer rate 99.9%, average summary length 83.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-pro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.949,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'amazon/nova-pro-v1:0': factual consistency rate 94.9% (hallucination rate 5.1%), answer rate 99.3%, average summary length 66.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-2-lite",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.949,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'amazon/nova-2-lite-v1:0': factual consistency rate 94.9% (hallucination rate 5.1%), answer rate 99.6%, average summary length 94.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-small-3",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.949,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/mistral-small-2501': factual consistency rate 94.9% (hallucination rate 5.1%), answer rate 97.9%, average summary length 98.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.948,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'ibm-granite/granite-4.0-h-small': factual consistency rate 94.8% (hallucination rate 5.2%), answer rate 100.0%, average summary length 107.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.948,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemma-4-26b-a4b-it': factual consistency rate 94.8% (hallucination rate 5.2%), answer rate 99.8%, average summary length 67.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.947,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-V3.2-Exp': factual consistency rate 94.7% (hallucination rate 5.3%), answer rate 96.6%, average summary length 64.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-14b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.946,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-14b': factual consistency rate 94.6% (hallucination rate 5.4%), answer rate 99.9%, average summary length 111.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-micro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'amazon/nova-micro-v1:0': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 100.0%, average summary length 100.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-V3.1': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 94.5%, average summary length 63.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.4-mini-2026-03-17': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 100.0%, average summary length 54.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-1",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.944,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-4.1-2025-04-14': factual consistency rate 94.4% (hallucination rate 5.6%), answer rate 99.9%, average summary length 91.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-4b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.943,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-4b': factual consistency rate 94.3% (hallucination rate 5.7%), answer rate 99.9%, average summary length 104.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-32b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.941,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-32b': factual consistency rate 94.1% (hallucination rate 5.9%), answer rate 99.9%, average summary length 115.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-lite",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.939,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'amazon/nova-lite-v1:0': factual consistency rate 93.9% (hallucination rate 6.1%), answer rate 99.9%, average summary length 91.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.939,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-V3': factual consistency rate 93.9% (hallucination rate 6.1%), answer rate 97.5%, average summary length 81.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.937,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-V3.2': factual consistency rate 93.7% (hallucination rate 6.3%), answer rate 92.6%, average summary length 62.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-4b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.936,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemma-3-4b-it': factual consistency rate 93.6% (hallucination rate 6.4%), answer rate 67.3%, average summary length 77.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.931,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'CohereLabs/command-r-plus-08-2024': factual consistency rate 93.1% (hallucination rate 6.9%), answer rate 95.0%, average summary length 91.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-6-sol",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.935,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-6-sol': factual consistency rate 93.5% (hallucination rate 6.5%), answer rate 100.0%, average summary length 71.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "trinity-large-preview",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.931,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'arcee-ai/trinity-large-preview': factual consistency rate 93.1% (hallucination rate 6.9%), answer rate 99.0%, average summary length 117.3 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-4",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.93,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.4-2026-03-05': factual consistency rate 93.0% (hallucination rate 7.0%), answer rate 99.9%, average summary length 81.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.93,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-2.5-pro': factual consistency rate 93.0% (hallucination rate 7.0%), answer rate 99.1%, average summary length 106.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.927,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/ministral-3b-2410': factual consistency rate 92.7% (hallucination rate 7.3%), answer rate 99.9%, average summary length 167.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-27b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.926,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemma-3-27b-it': factual consistency rate 92.6% (hallucination rate 7.4%), answer rate 98.8%, average summary length 96.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-4-31b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.926,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemma-4-31b-it': factual consistency rate 92.6% (hallucination rate 7.4%), answer rate 100.0%, average summary length 75.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-6-astra",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.913,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-6-astra': factual consistency rate 91.3% (hallucination rate 8.7%), answer rate 100.0%, average summary length 148.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.926,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/ministral-8b-2410': factual consistency rate 92.6% (hallucination rate 7.4%), answer rate 99.9%, average summary length 196.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-4-scout",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.923,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'meta-llama/Llama-4-Scout-17B-16E-Instruct': factual consistency rate 92.3% (hallucination rate 7.7%), answer rate 99.0%, average summary length 137.3 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.922,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-2.5-flash': factual consistency rate 92.2% (hallucination rate 7.8%), answer rate 99.0%, average summary length 101.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-4-maverick",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.918,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8': factual consistency rate 91.8% (hallucination rate 8.2%), answer rate 100.0%, average summary length 106.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Served via Together AI as Meta's own official FP8 release of this checkpoint.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.917,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.4-pro-2026-03-05': factual consistency rate 91.7% (hallucination rate 8.3%), answer rate 100.0%, average summary length 148.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "minimax-m2-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.909,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'MiniMaxAI/minimax-m2p5': factual consistency rate 90.9% (hallucination rate 9.1%), answer rate 98.2%, average summary length 137.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "command-a",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'CohereLabs/command-a-03-2025': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 97.6%, average summary length 101.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.5': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 100.0%, average summary length 129.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-5-air",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'zai-org/GLM-4.5-AIR-FP8': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 98.1%, average summary length 70.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Served via Together AI as Z.ai's own official FP8 release of this checkpoint.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-235b-a22b': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 94.9%, average summary length 105.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3-next-80b-a3b-thinking': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 94.4%, average summary length 70.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-7-flash",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.907,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'zai-org/GLM-4.7-flash': factual consistency rate 90.7% (hallucination rate 9.3%), answer rate 91.6%, average summary length 71.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "aya-expanse-8b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.905,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'CohereLabs/c4ai-aya-expanse-8b': factual consistency rate 90.5% (hallucination rate 9.5%), answer rate 77.5%, average summary length 88.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-6",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.905,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'zai-org/GLM-4.6': factual consistency rate 90.5% (hallucination rate 9.5%), answer rate 94.5%, average summary length 77.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.904,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'nvidia/Nemotron-3-Nano-30B-A3B': factual consistency rate 90.4% (hallucination rate 9.6%), answer rate 99.6%, average summary length 104.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.904,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-4o-2024-08-06': factual consistency rate 90.4% (hallucination rate 9.6%), answer rate 93.8%, average summary length 86.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.902,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-haiku-4-5-20251001': factual consistency rate 90.2% (hallucination rate 9.8%), answer rate 99.5%, average summary length 115.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.899,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'zai-org/glm-5': factual consistency rate 89.9% (hallucination rate 10.1%), answer rate 99.7%, average summary length 74.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.897,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-sonnet-4-20250514': factual consistency rate 89.7% (hallucination rate 10.3%), answer rate 98.6%, average summary length 145.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.896,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-3.1-pro-preview': factual consistency rate 89.6% (hallucination rate 10.4%), answer rate 99.4%, average summary length 107.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-5-flash",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.895,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3.5-flash-2026-02-23': factual consistency rate 89.5% (hallucination rate 10.5%), answer rate 99.8%, average summary length 95.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.895,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3.5-35b-a3b': factual consistency rate 89.5% (hallucination rate 10.5%), answer rate 99.8%, average summary length 94.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-nano",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.895,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5-nano-2025-08-07': factual consistency rate 89.5% (hallucination rate 10.5%), answer rate 100.0%, average summary length 105.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.894,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-sonnet-4-6': factual consistency rate 89.4% (hallucination rate 10.6%), answer rate 99.9%, average summary length 114.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.894,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'ibm-granite/granite-3.3-8b-instruct': factual consistency rate 89.4% (hallucination rate 10.6%), answer rate 100.0%, average summary length 131.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.893,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3.5-plus-2026-02-15': factual consistency rate 89.3% (hallucination rate 10.7%), answer rate 99.8%, average summary length 92.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-2",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.892,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.2-high-2025-12-11': factual consistency rate 89.2% (hallucination rate 10.8%), answer rate 100.0%, average summary length 186.3 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Highest reasoning effort listed (high; a low-effort row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "kimi-k2-6",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.892,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'moonshotai/kimi-k2.6': factual consistency rate 89.2% (hallucination rate 10.8%), answer rate 99.7%, average summary length 116.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.891,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-opus-4-5-20251101': factual consistency rate 89.1% (hallucination rate 10.9%), answer rate 98.7%, average summary length 114.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "aya-expanse-32b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.891,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'CohereLabs/c4ai-aya-expanse-32b': factual consistency rate 89.1% (hallucination rate 10.9%), answer rate 99.8%, average summary length 112.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.888,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3.5-122b-a10b': factual consistency rate 88.8% (hallucination rate 11.2%), answer rate 99.8%, average summary length 86.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-r1",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.887,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-R1': factual consistency rate 88.7% (hallucination rate 11.3%), answer rate 97.0%, average summary length 93.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-7",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.883,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'zai-org/glm-4p7': factual consistency rate 88.3% (hallucination rate 11.7%), answer rate 99.8%, average summary length 70.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.882,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-opus-4-1-20250805': factual consistency rate 88.2% (hallucination rate 11.8%), answer rate 92.4%, average summary length 129.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.88,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-opus-4-20250514': factual consistency rate 88.0% (hallucination rate 12.0%), answer rate 91.0%, average summary length 123.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.88,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-sonnet-4-5-20250929': factual consistency rate 88.0% (hallucination rate 12.0%), answer rate 95.6%, average summary length 127.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.88,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-opus-4-7': factual consistency rate 88.0% (hallucination rate 12.0%), answer rate 98.0%, average summary length 149.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.879,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'qwen/qwen3.5-27b': factual consistency rate 87.9% (hallucination rate 12.1%), answer rate 99.8%, average summary length 94.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.876,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.6-sol': factual consistency rate 87.6% (hallucination rate 12.4%), answer rate 99.1%, average summary length 131.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-1",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.879,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5.1-high-2025-11-13': factual consistency rate 87.9% (hallucination rate 12.1%), answer rate 100.0%, average summary length 254.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Highest reasoning effort listed (high; a low-effort row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.878,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'anthropic/claude-opus-4-6': factual consistency rate 87.8% (hallucination rate 12.2%), answer rate 99.8%, average summary length 137.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mercury-2",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.877,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'inceptionlabs/mercury-2': factual consistency rate 87.7% (hallucination rate 12.3%), answer rate 100.0%, average summary length 149.1 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "minimax-m2-7",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.871,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'MiniMaxAI/minimax-m2p7': factual consistency rate 87.1% (hallucination rate 12.9%), answer rate 99.4%, average summary length 131.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-mini",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.871,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5-mini-2025-08-07': factual consistency rate 87.1% (hallucination rate 12.9%), answer rate 99.9%, average summary length 169.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.865,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-3-flash-preview': factual consistency rate 86.5% (hallucination rate 13.5%), answer rate 99.8%, average summary length 90.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.864,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-3-pro-preview': factual consistency rate 86.4% (hallucination rate 13.6%), answer rate 99.4%, average summary length 101.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "kimi-k2-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.858,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'moonshotai/Kimi-K2.5': factual consistency rate 85.8% (hallucination rate 14.2%), answer rate 92.2%, average summary length 112.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.858,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-oss-120b': factual consistency rate 85.8% (hallucination rate 14.2%), answer rate 99.9%, average summary length 135.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-large-3",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.855,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/mistral-3-large-2512': factual consistency rate 85.5% (hallucination rate 14.5%), answer rate 98.8%, average summary length 112.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.849,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/gpt-5-high-2025-08-07': factual consistency rate 84.9% (hallucination rate 15.1%), answer rate 99.9%, average summary length 162.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Highest reasoning effort listed (high; a minimal-effort row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "kimi-k2-0905",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.821,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'moonshotai/Kimi-K2-Instruct-0905': factual consistency rate 82.1% (hallucination rate 17.9%), answer rate 98.6%, average summary length 59.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o4-mini",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.814,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/o4-mini-high-2025-04-16': factual consistency rate 81.4% (hallucination rate 18.6%), answer rate 99.2%, average summary length 127.7 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Highest reasoning effort listed (high; a low-effort row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.808,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'xai-org/grok-4-1-fast-reasoning': factual consistency rate 80.8% (hallucination rate 19.2%), answer rate 99.7%, average summary length 99.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-3-14b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.806,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/ministral-3-14b-2512': factual consistency rate 80.6% (hallucination rate 19.4%), answer rate 99.6%, average summary length 135.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "grok-4-fast",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.798,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'xai-org/grok-4-fast-reasoning': factual consistency rate 79.8% (hallucination rate 20.2%), answer rate 99.5%, average summary length 173.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Reasoning mode (a non-reasoning row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-3-8b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.783,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/ministral-3-8b-2512': factual consistency rate 78.3% (hallucination rate 21.7%), answer rate 99.1%, average summary length 139.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o3-pro",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.767,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'openai/o3-pro': factual consistency rate 76.7% (hallucination rate 23.3%), answer rate 100.0%, average summary length 127.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-4-mini",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.765,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'microsoft/Phi-4-mini-instruct': factual consistency rate 76.5% (hallucination rate 23.5%), answer rate 92.5%, average summary length 420.2 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-3-3b",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.758,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/ministral-3-3b-2512': factual consistency rate 75.8% (hallucination rate 24.2%), answer rate 74.3%, average summary length 119.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.993,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.0-Flash-001': factual consistency rate 99.3% (hallucination rate 0.7%), answer rate 100.0%, average summary length 65.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-0-pro-exp-02-05",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.992,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.0-Pro-Exp': factual consistency rate 99.2% (hallucination rate 0.8%), answer rate 99.7%, average summary length 61.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o3-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.992,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI o3-mini-high': factual consistency rate 99.2% (hallucination rate 0.8%), answer rate 100.0%, average summary length 79.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Highest reasoning effort listed.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "kimi-k2",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.989,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Moonshot AI Kimi-K2-Instruct': factual consistency rate 98.9% (hallucination rate 1.1%), answer rate 89.5%, average summary length 73.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-pro-exp-03-25",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.989,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.5-Pro-Exp-0325': factual consistency rate 98.9% (hallucination rate 1.1%), answer rate 95.1%, average summary length 72.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-0-flash-lite-preview-02-05",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.988,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.0-Flash-Lite-Preview': factual consistency rate 98.8% (hallucination rate 1.2%), answer rate 99.5%, average summary length 60.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.988,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4.5-Preview': factual consistency rate 98.8% (hallucination rate 1.2%), answer rate 100.0%, average summary length 77.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.987,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Zhipu AI GLM-4-9B-Chat': factual consistency rate 98.7% (hallucination rate 1.3%), answer rate 100.0%, average summary length 58.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-flash-preview-04-17",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.987,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.5-Flash-Preview': factual consistency rate 98.7% (hallucination rate 1.3%), answer rate 91.2%, average summary length 71.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.986,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-5-high': factual consistency rate 98.6% (hallucination rate 1.4%), answer rate 99.3%, average summary length 96.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Highest reasoning effort listed (a minimal-effort row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o1-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.986,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI-o1-mini': factual consistency rate 98.6% (hallucination rate 1.4%), answer rate 100.0%, average summary length 78.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-micro",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.984,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Amazon Nova-Micro-V1': factual consistency rate 98.4% (hallucination rate 1.6%), answer rate 100.0%, average summary length 90.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Called through OpenRouter, whose only provider for Nova is Amazon Bedrock.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.983,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4o-mini': factual consistency rate 98.3% (hallucination rate 1.7%), answer rate 100.0%, average summary length 76.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.983,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4-Turbo': factual consistency rate 98.3% (hallucination rate 1.7%), answer rate 100.0%, average summary length 86.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-lite",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.982,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Amazon Nova-Lite-V1': factual consistency rate 98.2% (hallucination rate 1.8%), answer rate 99.9%, average summary length 80.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Called through OpenRouter, whose only provider for Nova is Amazon Bedrock.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-0613",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.982,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4': factual consistency rate 98.2% (hallucination rate 1.8%), answer rate 100.0%, average summary length 81.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Called as the gpt-4 alias, which points to gpt-4-0613.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "nova-pro",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.982,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Amazon Nova-Pro-V1': factual consistency rate 98.2% (hallucination rate 1.8%), answer rate 100.0%, average summary length 85.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Called through OpenRouter, whose only provider for Nova is Amazon Bedrock.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-5-air",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.981,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'ZhipuAI GLM-4.5-Air-FP8': factual consistency rate 98.1% (hallucination rate 1.9%), answer rate 99.1%, average summary length 74.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Z.ai's own official FP8 release, served via Together AI.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.98,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4.1-nano': factual consistency rate 98.0% (hallucination rate 2.0%), answer rate 100.0%, average summary length 70.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.98,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4.1': factual consistency rate 98.0% (hallucination rate 2.0%), answer rate 100.0%, average summary length 71.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.978,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-4.1-mini': factual consistency rate 97.8% (hallucination rate 2.2%), answer rate 100.0%, average summary length 79.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-14b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.978,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-14B': factual consistency rate 97.8% (hallucination rate 2.2%), answer rate 100.0%, average summary length 82.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.976,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI gpt-oss-120b': factual consistency rate 97.6% (hallucination rate 2.4%), answer rate 100.0%, average summary length 82.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.976,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI o1': factual consistency rate 97.6% (hallucination rate 2.4%), answer rate 99.9%, average summary length 73.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.975,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Phi-3.5-MoE-instruct': factual consistency rate 97.5% (hallucination rate 2.5%), answer rate 96.3%, average summary length 69.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-pro-preview-06-05",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.974,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.5-Pro-Preview-06-05': factual consistency rate 97.4% (hallucination rate 2.6%), answer rate 99.6%, average summary length 73.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-4b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.973,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-4B': factual consistency rate 97.3% (hallucination rate 2.7%), answer rate 100.0%, average summary length 87.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-12b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.972,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-3-12B-Instruct': factual consistency rate 97.2% (hallucination rate 2.8%), answer rate 100.0%, average summary length 69.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.972,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-7B-Instruct': factual consistency rate 97.2% (hallucination rate 2.8%), answer rate 100.0%, average summary length 71.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-32b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.972,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-32B': factual consistency rate 97.2% (hallucination rate 2.8%), answer rate 100.0%, average summary length 82.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.971,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-2.5-Flash-Lite': factual consistency rate 97.1% (hallucination rate 2.9%), answer rate 99.7%, average summary length 78.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.971,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'AI21 Jamba-1.5-Mini': factual consistency rate 97.1% (hallucination rate 2.9%), answer rate 95.6%, average summary length 74.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-27b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.97,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-3-27B-Instruct': factual consistency rate 97.0% (hallucination rate 3.0%), answer rate 100.0%, average summary length 62.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.97,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-32B-Instruct': factual consistency rate 97.0% (hallucination rate 3.0%), answer rate 100.0%, average summary length 67.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "snowflake-arctic-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.97,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Snowflake-Arctic-Instruct': factual consistency rate 97.0% (hallucination rate 3.0%), answer rate 100.0%, average summary length 68.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.97,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-8B': factual consistency rate 97.0% (hallucination rate 3.0%), answer rate 100.0%, average summary length 78.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.969,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Phi-3-mini-128k-instruct': factual consistency rate 96.9% (hallucination rate 3.1%), answer rate 100.0%, average summary length 60.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-small-3",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.969,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral Small3': factual consistency rate 96.9% (hallucination rate 3.1%), answer rate 100.0%, average summary length 74.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.968,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-5-mini': factual consistency rate 96.8% (hallucination rate 3.2%), answer rate 99.6%, average summary length 87.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o1-preview",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.967,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI o1-preview': factual consistency rate 96.7% (hallucination rate 3.3%), answer rate 100.0%, average summary length 119.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.966,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-1.5-Flash-002': factual consistency rate 96.6% (hallucination rate 3.4%), answer rate 99.9%, average summary length 59.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-4-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.966,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Phi-4-mini-instruct': factual consistency rate 96.6% (hallucination rate 3.4%), answer rate 100.0%, average summary length 69.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI gpt-oss-20b': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 99.4%, average summary length 90.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-next-80b-a3b-thinking': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 99.7%, average summary length 64.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-5",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'ZhipuAI GLM-4.5': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 99.1%, average summary length 81.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-4b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-3-4B-Instruct': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 100.0%, average summary length 63.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-0.6B': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 100.0%, average summary length 65.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.963,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row '01-AI Yi-1.5-34B-Chat': factual consistency rate 96.3% (hallucination rate 3.7%), answer rate 100.0%, average summary length 83.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-max-preview",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.962,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-Max-Preview': factual consistency rate 96.2% (hallucination rate 3.8%), answer rate 97.0%, average summary length 80.8 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.961,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-V3': factual consistency rate 96.1% (hallucination rate 3.9%), answer rate 100.0%, average summary length 88.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.96,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3.3-70B-Instruct': factual consistency rate 96.0% (hallucination rate 4.0%), answer rate 100.0%, average summary length 85.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.959,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Phi-3.5-mini-instruct': factual consistency rate 95.9% (hallucination rate 4.1%), answer rate 100.0%, average summary length 75.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-70b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.959,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3-70B-Chat-hf': factual consistency rate 95.9% (hallucination rate 4.1%), answer rate 99.2%, average summary length 68.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.958,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-Opus-4.1': factual consistency rate 95.8% (hallucination rate 4.2%), answer rate 98.2%, average summary length 107.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.958,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2-VL-7B-Instruct': factual consistency rate 95.8% (hallucination rate 4.2%), answer rate 100.0%, average summary length 73.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.958,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-14B-Instruct': factual consistency rate 95.8% (hallucination rate 4.2%), answer rate 100.0%, average summary length 74.8 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.957,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-72B-Instruct': factual consistency rate 95.7% (hallucination rate 4.3%), answer rate 100.0%, average summary length 80.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "glm-4-6",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.957,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'ZhipuAI GLM-4.6': factual consistency rate 95.7% (hallucination rate 4.3%), answer rate 98.3%, average summary length 64.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.956,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-1.7B': factual consistency rate 95.6% (hallucination rate 4.4%), answer rate 100.0%, average summary length 69.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.955,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Claude-Haiku-4-5': factual consistency rate 95.5% (hallucination rate 4.5%), answer rate 99.7%, average summary length 83.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.955,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Claude-4-Sonnet': factual consistency rate 95.5% (hallucination rate 4.5%), answer rate 100.0%, average summary length 116.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.955,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Claude-3.7-Sonnet-Think': factual consistency rate 95.5% (hallucination rate 4.5%), answer rate 99.8%, average summary length 99.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Extended-thinking row (a non-thinking row is also published).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "command-a",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.955,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Cohere Command-A': factual consistency rate 95.5% (hallucination rate 4.5%), answer rate 100.0%, average summary length 77.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.954,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-AI DeepSeek-V3.2-Exp': factual consistency rate 95.4% (hallucination rate 4.6%), answer rate 99.3%, average summary length 61.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o4-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.954,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI o4-mini': factual consistency rate 95.4% (hallucination rate 4.6%), answer rate 100.0%, average summary length 82.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.954,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-3-5-sonnet': factual consistency rate 95.4% (hallucination rate 4.6%), answer rate 100.0%, average summary length 95.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gpt-5-nano",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.953,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI GPT-5-nano': factual consistency rate 95.3% (hallucination rate 4.7%), answer rate 99.9%, average summary length 72.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-72b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.953,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2-72B-Instruct': factual consistency rate 95.3% (hallucination rate 4.7%), answer rate 100.0%, average summary length 100.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-4",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.953,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Phi-4': factual consistency rate 95.3% (hallucination rate 4.7%), answer rate 100.0%, average summary length 100.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.953,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mixtral-8x22B-Instruct-v0.1': factual consistency rate 95.3% (hallucination rate 4.7%), answer rate 99.9%, average summary length 92.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.952,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Claude-4-Opus': factual consistency rate 95.2% (hallucination rate 4.8%), answer rate 100.0%, average summary length 110.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.952,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen3-30b-a3b-thinking-2507': factual consistency rate 95.2% (hallucination rate 4.8%), answer rate 98.2%, average summary length 68.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "grok-4-0709",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.952,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'XAI Grok-4-0709': factual consistency rate 95.2% (hallucination rate 4.8%), answer rate 99.8%, average summary length 100.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.951,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-3-5-haiku': factual consistency rate 95.1% (hallucination rate 4.9%), answer rate 100.0%, average summary length 92.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.951,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row '01-AI Yi-1.5-9B-Chat': factual consistency rate 95.1% (hallucination rate 4.9%), answer rate 100.0%, average summary length 85.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "command-r-08-2024",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.951,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Cohere Command-R': factual consistency rate 95.1% (hallucination rate 4.9%), answer rate 100.0%, average summary length 68.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.95,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3.1-70B-Instruct': factual consistency rate 95.0% (hallucination rate 5.0%), answer rate 100.0%, average summary length 79.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-3-1b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.947,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-3-1B-Instruct': factual consistency rate 94.7% (hallucination rate 5.3%), answer rate 99.9%, average summary length 57.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.946,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3.1-8B-Instruct': factual consistency rate 94.6% (hallucination rate 5.4%), answer rate 100.0%, average summary length 71.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.946,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Cohere Command-R-Plus': factual consistency rate 94.6% (hallucination rate 5.4%), answer rate 100.0%, average summary length 68.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-Sonnet-4-5-20250929': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 99.4%, average summary length 87.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-AI DeepSeek-V3.1': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 98.7%, average summary length 78.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.944,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral-Small-3.1-24B-Instruct': factual consistency rate 94.4% (hallucination rate 5.6%), answer rate 100.0%, average summary length 73.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.941,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-2-70B-Chat-hf': factual consistency rate 94.1% (hallucination rate 5.9%), answer rate 99.9%, average summary length 84.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "kimi-k2-0905",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.938,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'MoonshotAI Kimi-K2-Instruct-0905': factual consistency rate 93.8% (hallucination rate 6.2%), answer rate 99.6%, average summary length 61.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.935,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.0-8B-Instruct': factual consistency rate 93.5% (hallucination rate 6.5%), answer rate 100.0%, average summary length 74.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-4-0-micro",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.935,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-4.0-micro': factual consistency rate 93.5% (hallucination rate 6.5%), answer rate 100.0%, average summary length 59.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.937,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-1.5-Pro-002': factual consistency rate 93.7% (hallucination rate 6.6%), answer rate 99.9%, average summary length 62.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.934,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-1.5-Flash': factual consistency rate 93.4% (hallucination rate 6.6%), answer rate 99.9%, average summary length 63.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "phi-2",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.933,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft phi-2': factual consistency rate 93.3% (hallucination rate 6.7%), answer rate 91.5%, average summary length 80.8 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "o3",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.932,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'OpenAI o3': factual consistency rate 93.2% (hallucination rate 6.8%), answer rate 100.0%, average summary length 77.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-2-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.93,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-2-2B-it': factual consistency rate 93.0% (hallucination rate 7.0%), answer rate 100.0%, average summary length 62.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.93,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-3B-Instruct': factual consistency rate 93.0% (hallucination rate 7.0%), answer rate 100.0%, average summary length 70.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.926,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3-8B-Chat-hf': factual consistency rate 92.6% (hallucination rate 7.4%), answer rate 99.8%, average summary length 79.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.925,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral-Ministral-8B': factual consistency rate 92.5% (hallucination rate 7.5%), answer rate 100.0%, average summary length 62.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.923,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-R1-0528': factual consistency rate 92.3% (hallucination rate 7.7%), answer rate 100.0%, average summary length 138.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.921,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row '01-AI Yi-1.5-6B-Chat': factual consistency rate 92.1% (hallucination rate 7.9%), answer rate 100.0%, average summary length 98.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.92,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-V3-0324': factual consistency rate 92.0% (hallucination rate 8.0%), answer rate 100.0%, average summary length 78.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.917,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral-Ministral-3B': factual consistency rate 91.7% (hallucination rate 8.3%), answer rate 100.0%, average summary length 73.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "dbrx-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.917,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'databricks dbrx-instruct': factual consistency rate 91.7% (hallucination rate 8.3%), answer rate 100.0%, average summary length 85.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "aya-expanse-32b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.915,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Cohere Aya Expanse 32B': factual consistency rate 91.5% (hallucination rate 8.5%), answer rate 99.9%, average summary length 81.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.914,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.1-8B-Instruct': factual consistency rate 91.4% (hallucination rate 8.6%), answer rate 100.0%, average summary length 107.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-2-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.913,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.2-8B-Instruct': factual consistency rate 91.3% (hallucination rate 8.7%), answer rate 100.0%, average summary length 120.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.912,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.0-2B-Instruct': factual consistency rate 91.2% (hallucination rate 8.8%), answer rate 100.0%, average summary length 81.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.905,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral-7B-Instruct-v0.3': factual consistency rate 90.5% (hallucination rate 9.5%), answer rate 100.0%, average summary length 98.4 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.909,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemini-1.5-Pro': factual consistency rate 90.9% (hallucination rate 9.1%), answer rate 99.8%, average summary length 61.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.899,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-3-opus': factual consistency rate 89.9% (hallucination rate 10.1%), answer rate 95.5%, average summary length 92.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-2-9b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.899,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-2-9B-it': factual consistency rate 89.9% (hallucination rate 10.1%), answer rate 100.0%, average summary length 70.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.895,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-2-13B-Chat-hf': factual consistency rate 89.5% (hallucination rate 10.5%), answer rate 99.8%, average summary length 82.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "olmo-2-13b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.892,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'AllenAI-OLMo-2-13B-Instruct': factual consistency rate 89.2% (hallucination rate 10.8%), answer rate 100.0%, average summary length 82.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "olmo-2-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.889,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'AllenAI-OLMo-2-7B-Instruct': factual consistency rate 88.9% (hallucination rate 11.1%), answer rate 100.0%, average summary length 112.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.888,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mistral-Nemo-Instruct': factual consistency rate 88.8% (hallucination rate 11.2%), answer rate 100.0%, average summary length 69.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.887,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-2-7B-Chat-hf': factual consistency rate 88.7% (hallucination rate 11.3%), answer rate 99.6%, average summary length 119.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mistral-medium-3-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.886,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'MistralAI Mistral-Medium-2508': factual consistency rate 88.6% (hallucination rate 11.4%), answer rate 98.3%, average summary length 99.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "wizardlm-2-8x22b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.883,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft WizardLM-2-8x22B': factual consistency rate 88.3% (hallucination rate 11.7%), answer rate 99.9%, average summary length 140.8 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "aya-expanse-8b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.878,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Cohere Aya Expanse 8B': factual consistency rate 87.8% (hallucination rate 12.2%), answer rate 99.9%, average summary length 83.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-r1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.857,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'DeepSeek-R1': factual consistency rate 85.7% (hallucination rate 14.3%), answer rate 100.0%, average summary length 77.1 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.852,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-7B-it': factual consistency rate 85.2% (hallucination rate 14.8%), answer rate 100.0%, average summary length 113.0 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.843,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.1-2B-Instruct': factual consistency rate 84.3% (hallucination rate 15.7%), answer rate 100.0%, average summary length 107.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.842,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-1.5B-Instruct': factual consistency rate 84.2% (hallucination rate 15.8%), answer rate 100.0%, average summary length 70.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.839,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen-QwQ-32B-Preview': factual consistency rate 83.9% (hallucination rate 16.1%), answer rate 100.0%, average summary length 201.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.837,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-3-sonnet': factual consistency rate 83.7% (hallucination rate 16.3%), answer rate 100.0%, average summary length 108.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-3-2-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.835,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-3.2-2B-Instruct': factual consistency rate 83.5% (hallucination rate 16.5%), answer rate 100.0%, average summary length 117.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.83,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-1.1-7B-it': factual consistency rate 83.0% (hallucination rate 17.0%), answer rate 100.0%, average summary length 64.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-2-0",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.826,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Anthropic Claude-2': factual consistency rate 82.6% (hallucination rate 17.4%), answer rate 99.3%, average summary length 87.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "flan-t5-large",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.817,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Flan-T5-large': factual consistency rate 81.7% (hallucination rate 18.3%), answer rate 99.3%, average summary length 20.9 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.799,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Mixtral-8x7B-Instruct-v0.1': factual consistency rate 79.9% (hallucination rate 20.1%), answer rate 99.9%, average summary length 90.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.793,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Llama-3.2-1B-Instruct': factual consistency rate 79.3% (hallucination rate 20.7%), answer rate 100.0%, average summary length 71.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.748,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2.5-0.5B-Instruct': factual consistency rate 74.8% (hallucination rate 25.2%), answer rate 100.0%, average summary length 72.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.722,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Google Gemma-1.1-2B-it': factual consistency rate 72.2% (hallucination rate 27.8%), answer rate 100.0%, average summary length 66.8 words. HHEM-2.3 judge; the model identity follows the README's API integration details.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau3-airline",
    "score": 0.84,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.5', domain airline, pass^1 = 84.00% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau3-retail",
    "score": 0.7961,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.5', domain retail, pass^1 = 79.61% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau3-telecom",
    "score": 0.9232,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.5', domain telecom, pass^1 = 92.32% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau3-banking-v101",
    "score": 0.2474,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.5', domain banking_knowledge, pass^1 = 24.74% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau3-airline",
    "score": 0.72,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-02-24",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Sonnet 4.5', domain airline, pass^1 = 72.00% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-24. Notes: Evaluated using Claude Sonnet 4.5 (claude-sonnet-4-5-20250929) with extended thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking domain evaluated with terminal-based agentic search retrieval. banking_knowledge re-graded under tau2-bench v1.0.1 grading f",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau3-retail",
    "score": 0.7237,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-02-24",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Sonnet 4.5', domain retail, pass^1 = 72.37% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-24. Notes: Evaluated using Claude Sonnet 4.5 (claude-sonnet-4-5-20250929) with extended thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking domain evaluated with terminal-based agentic search retrieval. banking_knowledge re-graded under tau2-bench v1.0.1 grading f",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau3-telecom",
    "score": 0.8487,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-02-24",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Sonnet 4.5', domain telecom, pass^1 = 84.87% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-24. Notes: Evaluated using Claude Sonnet 4.5 (claude-sonnet-4-5-20250929) with extended thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking domain evaluated with terminal-based agentic search retrieval. banking_knowledge re-graded under tau2-bench v1.0.1 grading f",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau3-banking-v101",
    "score": 0.2526,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-02-24",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Sonnet 4.5', domain banking_knowledge, pass^1 = 25.26% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-24. Notes: Evaluated using Claude Sonnet 4.5 (claude-sonnet-4-5-20250929) with extended thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking domain evaluated with terminal-based agentic search retrieval. banking_knowledge re-graded under tau2-bench v1.0.1 grading f",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "tau3-airline",
    "score": 0.825,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-flash_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-flash_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Flash', domain airline, pass^1 = 82.50% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-flash-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 123. Telecom domain had 20 simulations terminated due to too_many_errors (all scored as 0). Banking domain had 52 simulations terminated due to max_steps ",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "tau3-retail",
    "score": 0.7675,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-flash_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-flash_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Flash', domain retail, pass^1 = 76.75% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-flash-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 123. Telecom domain had 20 simulations terminated due to too_many_errors (all scored as 0). Banking domain had 52 simulations terminated due to max_steps ",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "tau3-telecom",
    "score": 0.9123,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-flash_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-flash_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Flash', domain telecom, pass^1 = 91.23% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-flash-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 123. Telecom domain had 20 simulations terminated due to too_many_errors (all scored as 0). Banking domain had 52 simulations terminated due to max_steps ",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "tau3-banking-v101",
    "score": 0.2732,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-flash_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-flash_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Flash', domain banking_knowledge, pass^1 = 27.32% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-flash-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 123. Telecom domain had 20 simulations terminated due to too_many_errors (all scored as 0). Banking domain had 52 simulations terminated due to max_steps ",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau3-airline",
    "score": 0.805,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-pro_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-pro_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Pro', domain airline, pass^1 = 80.50% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-pro-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Infrastructure errors treated as failures (reward 0): 3 in airline, 20 in retail, 2 in telecom. Banking domain had 42 simulations terminated due to max_steps (all",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau3-retail",
    "score": 0.7588,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-pro_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-pro_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Pro', domain retail, pass^1 = 75.88% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-pro-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Infrastructure errors treated as failures (reward 0): 3 in airline, 20 in retail, 2 in telecom. Banking domain had 42 simulations terminated due to max_steps (all",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau3-telecom",
    "score": 0.9101,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-pro_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-pro_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Pro', domain telecom, pass^1 = 91.01% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-pro-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Infrastructure errors treated as failures (reward 0): 3 in airline, 20 in retail, 2 in telecom. Banking domain had 42 simulations terminated due to max_steps (all",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau3-banking-v101",
    "score": 0.1804,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-pro_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-03-02",
    "note": "τ²-bench leaderboard submission 'gemini-3-pro_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3 Pro', domain banking_knowledge, pass^1 = 18.04% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-03-02. Notes: Evaluated using vertex_ai/gemini-3-pro-preview with reasoning_effort: high. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Infrastructure errors treated as failures (reward 0): 3 in airline, 20 in retail, 2 in telecom. Banking domain had 42 simulations terminated due to max_steps (all",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "glm-5",
    "bench": "tau3-airline",
    "score": 0.825,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/glm-5-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'glm-5-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GLM-5', domain airline, pass^1 = 82.50% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using GLM-5 with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Infrastructure errors treated as failures. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, nu",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "glm-5",
    "bench": "tau3-retail",
    "score": 0.7368,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/glm-5-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'glm-5-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GLM-5', domain retail, pass^1 = 73.68% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using GLM-5 with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Infrastructure errors treated as failures. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, nu",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "glm-5",
    "bench": "tau3-telecom",
    "score": 0.8684,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/glm-5-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'glm-5-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GLM-5', domain telecom, pass^1 = 86.84% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using GLM-5 with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Infrastructure errors treated as failures. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, nu",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "glm-5",
    "bench": "tau3-banking-v101",
    "score": 0.0979,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/glm-5-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'glm-5-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GLM-5', domain banking_knowledge, pass^1 = 9.79% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using GLM-5 with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Infrastructure errors treated as failures. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, nu",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau3-airline",
    "score": 0.83,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-2_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gpt-5-2_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.2', domain airline, pass^1 = 83.00% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau3-retail",
    "score": 0.8158,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-2_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gpt-5-2_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.2', domain retail, pass^1 = 81.58% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau3-telecom",
    "score": 0.8969,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-2_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gpt-5-2_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.2', domain telecom, pass^1 = 89.69% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau3-banking-v101",
    "score": 0.3222,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-2_sierra_2026-02-26/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gpt-5-2_sierra_2026-02-26' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.2', domain banking_knowledge, pass^1 = 32.22% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-airline",
    "score": 0.815,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'qwen3.5-397b-a17b-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Qwen3.5-397B-A17B', domain airline, pass^1 = 81.50% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using Qwen3.5-397B-A17B with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Self-hosted via vLLM. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, numeric-arg",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-retail",
    "score": 0.8443,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'qwen3.5-397b-a17b-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Qwen3.5-397B-A17B', domain retail, pass^1 = 84.43% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using Qwen3.5-397B-A17B with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Self-hosted via vLLM. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, numeric-arg",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-telecom",
    "score": 0.9781,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'qwen3.5-397b-a17b-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Qwen3.5-397B-A17B', domain telecom, pass^1 = 97.81% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using Qwen3.5-397B-A17B with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Self-hosted via vLLM. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, numeric-arg",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-banking-v101",
    "score": 0.0979,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json",
    "by": "independent",
    "date": "2026-02-27",
    "note": "τ²-bench leaderboard submission 'qwen3.5-397b-a17b-think_sierra_2026-03-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Qwen3.5-397B-A17B', domain banking_knowledge, pass^1 = 9.79% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort enabled, evaluated 2026-02-27. Notes: Evaluated using Qwen3.5-397B-A17B with thinking enabled. User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Temperature: 1.0, top_p: 0.95. Self-hosted via vLLM. banking_knowledge re-graded under tau2-bench v1.0.1 grading fixes on 2026-07-15 (extra-read logging #329, numeric-arg",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-fable-5",
    "bench": "tau3-banking-v101",
    "score": 0.3969,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-fable-5_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-23",
    "note": "τ²-bench leaderboard submission 'claude-fable-5_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Fable 5', domain banking_knowledge, pass^1 = 39.69% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-07-23. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "tau3-banking-v101",
    "score": 0.2732,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-6_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-06",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-6_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.6', domain banking_knowledge, pass^1 = 27.32% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-05-06. Notes: Evaluated with reasoning_effort 'max' (the highest effort level this model supports). Retrieval: AllTools (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentional",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "tau3-banking-v101",
    "score": 0.4021,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-7_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-07-23",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-7_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.7', domain banking_knowledge, pass^1 = 40.21% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-07-23. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "tau3-banking-v101",
    "score": 0.3969,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-8_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-23",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-8_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 4.8', domain banking_knowledge, pass^1 = 39.69% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-07-23. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-5",
    "bench": "tau3-banking-v101",
    "score": 0.4871,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-5_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-08-03",
    "note": "τ²-bench leaderboard submission 'claude-opus-5_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude Opus 5', domain banking_knowledge, pass^1 = 48.71% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-08-03. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "tau3-banking-v101",
    "score": 0.1366,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-2-5-pro_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gemini-2-5-pro_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 2.5 Pro', domain banking_knowledge, pass^1 = 13.66% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "tau3-banking-v101",
    "score": 0.2603,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gemini-3-1-pro-preview_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'gemini-3-1-pro-preview_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Gemini 3.1 Pro Preview', domain banking_knowledge, pass^1 = 26.03% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "glm-5-2",
    "bench": "tau3-banking-v101",
    "score": 0.3711,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/glm-5-2_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-24",
    "note": "τ²-bench leaderboard submission 'glm-5-2_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GLM-5.2', domain banking_knowledge, pass^1 = 37.11% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-07-24. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-4",
    "bench": "tau3-banking-v101",
    "score": 0.3943,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-4_sierra_2026-03-25/submission.json",
    "by": "independent",
    "date": "2026-05-06",
    "note": "τ²-bench leaderboard submission 'gpt-5-4_sierra_2026-03-25' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.4', domain banking_knowledge, pass^1 = 39.43% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-05-06. Notes: Evaluated with reasoning_effort 'xhigh' (the highest effort level this model supports). Retrieval: AllTools (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intention",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-5",
    "bench": "tau3-banking-v101",
    "score": 0.4459,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-5_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-07-22",
    "note": "τ²-bench leaderboard submission 'gpt-5-5_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.5', domain banking_knowledge, pass^1 = 44.59% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-07-22. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "tau3-banking-v101",
    "score": 0.4691,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5-6-sol_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-22",
    "note": "τ²-bench leaderboard submission 'gpt-5-6-sol_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5.6-sol', domain banking_knowledge, pass^1 = 46.91% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-07-22. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "tau3-banking-v101",
    "score": 0.1314,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/grok-4-1-fast_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'grok-4-1-fast_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Grok 4.1 fast', domain banking_knowledge, pass^1 = 13.14% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "grok-4-20",
    "bench": "tau3-banking-v101",
    "score": 0.1804,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/grok-4-2_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'grok-4-2_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Grok 4.2', domain banking_knowledge, pass^1 = 18.04% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "grok-4-5",
    "bench": "tau3-banking-v101",
    "score": 0.4794,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/grok-4-5_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-24",
    "note": "τ²-bench leaderboard submission 'grok-4-5_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Grok 4.5', domain banking_knowledge, pass^1 = 47.94% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-07-24. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "grok-4-fast",
    "bench": "tau3-banking-v101",
    "score": 0.1572,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/grok-4-fast_sierra_2026-05-05/submission.json",
    "by": "independent",
    "date": "2026-05-05",
    "note": "τ²-bench leaderboard submission 'grok-4-fast_sierra_2026-05-05' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Grok 4 fast', domain banking_knowledge, pass^1 = 15.72% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort high, evaluated 2026-05-05. Notes: Evaluated using AllTools retrieval (BM25 + dense OpenAI text-embedding-3-large + sandboxed shell). User simulator: gpt-5.2 with reasoning_effort: low. 4 trials. Seed: 300. Banking_knowledge domain only — other domains intentionally excluded from this comparison; the AllTools setting standardizes ret",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "inkling",
    "bench": "tau3-banking-v101",
    "score": 0.25,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/inkling_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-24",
    "note": "τ²-bench leaderboard submission 'inkling_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Inkling', domain banking_knowledge, pass^1 = 25.00% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-07-24. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "kimi-k3",
    "bench": "tau3-banking-v101",
    "score": 0.3711,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/kimi-k3_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-24",
    "note": "τ²-bench leaderboard submission 'kimi-k3_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Kimi K3', domain banking_knowledge, pass^1 = 37.11% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort max, evaluated 2026-07-24. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "muse-spark-1-1",
    "bench": "tau3-banking-v101",
    "score": 0.4046,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/muse-spark-1-1_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-07-23",
    "note": "τ²-bench leaderboard submission 'muse-spark-1-1_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Muse Spark 1.1', domain banking_knowledge, pass^1 = 40.46% (stored /100). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-07-23. Notes: Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "tau2-retail",
    "score": 0.721,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-3-7-sonnet_anthropic_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'claude-3-7-sonnet_anthropic_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude-3.7-Sonnet', domain retail, pass^1 = 72.10% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "tau2-airline",
    "score": 0.642,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-3-7-sonnet_anthropic_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'claude-3-7-sonnet_anthropic_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude-3.7-Sonnet', domain airline, pass^1 = 64.20% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "tau2-telecom",
    "score": 0.49,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-3-7-sonnet_anthropic_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'claude-3-7-sonnet_anthropic_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Claude-3.7-Sonnet', domain telecom, pass^1 = 49.00% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-retail",
    "score": 0.74,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1', domain retail, pass^1 = 74.00% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-airline",
    "score": 0.56,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1', domain airline, pass^1 = 56.00% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-telecom",
    "score": 0.34,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1', domain telecom, pass^1 = 34.00% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "tau2-retail",
    "score": 0.614,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1-mini', domain retail, pass^1 = 61.40% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "tau2-airline",
    "score": 0.487,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1-mini', domain airline, pass^1 = 48.70% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "tau2-telecom",
    "score": 0.489,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-4-1-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'gpt-4-1-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-4.1-mini', domain telecom, pass^1 = 48.90% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5",
    "bench": "tau2-retail",
    "score": 0.8158,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5_sierra_2025-08-09/submission.json",
    "by": "independent",
    "date": "2025-08-09",
    "note": "τ²-bench leaderboard submission 'gpt-5_sierra_2025-08-09' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5', domain retail, pass^1 = 81.58% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-08-09.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5",
    "bench": "tau2-airline",
    "score": 0.625,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5_sierra_2025-08-09/submission.json",
    "by": "independent",
    "date": "2025-08-09",
    "note": "τ²-bench leaderboard submission 'gpt-5_sierra_2025-08-09' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5', domain airline, pass^1 = 62.50% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-08-09.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "gpt-5",
    "bench": "tau2-telecom",
    "score": 0.9583,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/gpt-5_sierra_2025-08-09/submission.json",
    "by": "independent",
    "date": "2025-08-09",
    "note": "τ²-bench leaderboard submission 'gpt-5_sierra_2025-08-09' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'GPT-5', domain telecom, pass^1 = 95.83% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-08-09.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-retail",
    "score": 0.683,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/o4-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'o4-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'o4-mini', domain retail, pass^1 = 68.30% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-airline",
    "score": 0.521,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/o4-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'o4-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'o4-mini', domain airline, pass^1 = 52.10% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-telecom",
    "score": 0.502,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/o4-mini_openai_2024-06-20/submission.json",
    "by": "independent",
    "date": "2025-06-09",
    "note": "τ²-bench leaderboard submission 'o4-mini_openai_2024-06-20' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'o4-mini', domain telecom, pass^1 = 50.20% (stored /100). tau2-bench version 0.1.3, user simulator gpt-4.1-2025-04-14, reasoning effort None, evaluated 2025-06-09. Notes: Original evaluation from tau2-bench paper",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "tau-bench-retail",
    "score": 0.824,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-1_anthropic_2025-01-15/submission.json",
    "by": "lab",
    "date": "2025-08-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-1_anthropic_2025-01-15' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Opus 4.1', domain retail, pass^1 = 82.40% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-08-05. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "tau-bench-airline",
    "score": 0.56,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4-1_anthropic_2025-01-15/submission.json",
    "by": "lab",
    "date": "2025-08-05",
    "note": "τ²-bench leaderboard submission 'claude-opus-4-1_anthropic_2025-01-15' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Opus 4.1', domain airline, pass^1 = 56.00% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-08-05. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "tau-bench-retail",
    "score": 0.814,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4_anthropic_2025-05-22/submission.json",
    "by": "lab",
    "date": "2025-05-22",
    "note": "τ²-bench leaderboard submission 'claude-opus-4_anthropic_2025-05-22' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Opus 4', domain retail, pass^1 = 81.40% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-05-22. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "tau-bench-airline",
    "score": 0.596,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-opus-4_anthropic_2025-05-22/submission.json",
    "by": "lab",
    "date": "2025-05-22",
    "note": "τ²-bench leaderboard submission 'claude-opus-4_anthropic_2025-05-22' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Opus 4', domain airline, pass^1 = 59.60% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-05-22. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau-bench-retail",
    "score": 0.805,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4_anthropic_2025-05-22/submission.json",
    "by": "lab",
    "date": "2025-05-22",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4_anthropic_2025-05-22' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Sonnet 4', domain retail, pass^1 = 80.50% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-05-22. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau-bench-airline",
    "score": 0.6,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4_anthropic_2025-05-22/submission.json",
    "by": "lab",
    "date": "2025-05-22",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4_anthropic_2025-05-22' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude Sonnet 4', domain airline, pass^1 = 60.00% (stored /100). tau2-bench version original tau-bench, user simulator None, reasoning effort None, evaluated 2025-05-22. Notes: Scores were achieved with a prompt addendum to both the Airline and Retail Agent Policy instructing Claude to better leverage its reasoning abilities while using extended thinking with tool use. The model is encouraged to write down its thoughts as it solves the problem distinct from our usual think",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-retail",
    "score": 0.862,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_anthropic_2025-10-02/submission.json",
    "by": "lab",
    "date": "2025-09-29",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_anthropic_2025-10-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude-Sonnet-4.5', domain retail, pass^1 = 86.20% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-09-29. Notes: Scores were achieved using extended thinking with tool use and a prompt addendum to the Airline and Telecom Agent Policy instructing Claude to better target its known failure modes when using the vanilla prompt. A prompt addendum was also added to the Telecom User prompt to avoid failure modes from ",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-airline",
    "score": 0.7,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_anthropic_2025-10-02/submission.json",
    "by": "lab",
    "date": "2025-09-29",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_anthropic_2025-10-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude-Sonnet-4.5', domain airline, pass^1 = 70.00% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-09-29. Notes: Scores were achieved using extended thinking with tool use and a prompt addendum to the Airline and Telecom Agent Policy instructing Claude to better target its known failure modes when using the vanilla prompt. A prompt addendum was also added to the Telecom User prompt to avoid failure modes from ",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "tau2-telecom",
    "score": 0.98,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/claude-sonnet-4-5_anthropic_2025-10-02/submission.json",
    "by": "lab",
    "date": "2025-09-29",
    "note": "τ²-bench leaderboard submission 'claude-sonnet-4-5_anthropic_2025-10-02' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Anthropic: model 'Claude-Sonnet-4.5', domain telecom, pass^1 = 98.00% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-09-29. Notes: Scores were achieved using extended thinking with tool use and a prompt addendum to the Airline and Telecom Agent Policy instructing Claude to better target its known failure modes when using the vanilla prompt. A prompt addendum was also added to the Telecom User prompt to avoid failure modes from ",
    "source_id": "tau2-bench-leaderboard-anthropic"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-retail",
    "score": 0.706,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/kimi-k2_moonshot-ai_2025-07-11/submission.json",
    "by": "lab",
    "date": "2025-07-28",
    "note": "τ²-bench leaderboard submission 'kimi-k2_moonshot-ai_2025-07-11' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Moonshot AI: model 'Kimi-k2', domain retail, pass^1 = 70.60% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-07-28. Notes: Evaluation limited to Pass^1 scores only",
    "source_id": "tau2-bench-leaderboard-moonshot"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-airline",
    "score": 0.565,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/kimi-k2_moonshot-ai_2025-07-11/submission.json",
    "by": "lab",
    "date": "2025-07-28",
    "note": "τ²-bench leaderboard submission 'kimi-k2_moonshot-ai_2025-07-11' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Moonshot AI: model 'Kimi-k2', domain airline, pass^1 = 56.50% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-07-28. Notes: Evaluation limited to Pass^1 scores only",
    "source_id": "tau2-bench-leaderboard-moonshot"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-telecom",
    "score": 0.658,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/kimi-k2_moonshot-ai_2025-07-11/submission.json",
    "by": "lab",
    "date": "2025-07-28",
    "note": "τ²-bench leaderboard submission 'kimi-k2_moonshot-ai_2025-07-11' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Moonshot AI: model 'Kimi-k2', domain telecom, pass^1 = 65.80% (stored /100). tau2-bench version 0.1.3, user simulator None, reasoning effort None, evaluated 2025-07-28. Notes: Evaluation limited to Pass^1 scores only",
    "source_id": "tau2-bench-leaderboard-moonshot"
  },
  {
    "model": "mistral-large-3",
    "bench": "mideind-icelandic-average",
    "score": 0.257014,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Mistral Large 3' (mistral/mistral-large-2512), thinking='none', gen_kwargs={\"max_tokens\": 32000}; highest of the listed settings ['none']. Published 'average' (Knowledge 0.181233, Language Proficiency 0.31385). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "mistral-small-4",
    "bench": "mideind-icelandic-average",
    "score": 0.206097,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-28",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Mistral Small 4' (mistral/mistral-small-2603), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"reasoning_effort\": \"high\"}}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.131699, Language Proficiency 0.261896). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "mideind-icelandic-average",
    "score": 0.148852,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-26",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Mistral Medium 3.5' (mistral/mistral-medium-2604), thinking='high', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"reasoning_effort\": \"high\"}}; highest of the listed settings ['high', 'off']. Published 'average' (Knowledge 0.12421, Language Proficiency 0.167334). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "picovoice-en4-streaming-avg-wer",
    "score": 0.078,
    "source": "https://github.com/Picovoice/speech-to-text-benchmark/blob/aea7914706d44d3163c1c0283e05700900abe7af/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Picovoice README 'English > Streaming Engines Word Error Rate', row 'Nemotron 3.5 ASR Streaming' (linked checkpoint nvidia/nemotron-3.5-asr-streaming-0.6b), column 'Average' = 7.8% (mean of LibriSpeech test-clean 3.4%, test-other 7.8%, TED-LIUM 4.9%, CommonVoice 15.0%); stored 0.078. Same table and normalisation as the existing streaming rows; the open-model rows in that table are unchanged at this revision.",
    "source_id": "speech-stt-picovoice-apache"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "charxiv-rq",
    "score": 0.602,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3.5 Sonnet', Reasoning 'Overall' = 60.20%; stored /100. Identity: July 2024 addition; claude-3-5-sonnet-20240620 was the only Claude 3.5 Sonnet snapshot then.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "charxiv-dq",
    "score": 0.843,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3.5 Sonnet', Descriptive 'Overall' = 84.30%; stored /100. Identity: July 2024 addition; claude-3-5-sonnet-20240620 was the only Claude 3.5 Sonnet snapshot then.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "charxiv-rq",
    "score": 0.471,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4o', Reasoning 'Overall' = 47.10%; stored /100. Identity: paper v1 names gpt-4o-2024-05-13.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "charxiv-dq",
    "score": 0.8445,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4o', Descriptive 'Overall' = 84.45%; stored /100. Identity: paper v1 names gpt-4o-2024-05-13.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "charxiv-rq",
    "score": 0.433,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Gemini 1.5 Pro', Reasoning 'Overall' = 43.30%; stored /100. Identity: July 2024 addition; gemini-1.5-pro-001 was the only Gemini 1.5 Pro GA version then.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "charxiv-dq",
    "score": 0.7197,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Gemini 1.5 Pro', Descriptive 'Overall' = 71.97%; stored /100. Identity: July 2024 addition; gemini-1.5-pro-001 was the only Gemini 1.5 Pro GA version then.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "charxiv-rq",
    "score": 0.371,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4V', Reasoning 'Overall' = 37.10%; stored /100. Identity: paper v1 names gpt-4-turbo-2024-04-09 (results file GPT-4V-Turbo).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "charxiv-dq",
    "score": 0.7992,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4V', Descriptive 'Overall' = 79.92%; stored /100. Identity: paper v1 names gpt-4-turbo-2024-04-09 (results file GPT-4V-Turbo).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "charxiv-rq",
    "score": 0.341,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4o Mini', Reasoning 'Overall' = 34.10%; stored /100. Identity: July 2024 addition; gpt-4o-mini-2024-07-18 was its only snapshot.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "charxiv-dq",
    "score": 0.7492,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'GPT-4o Mini', Descriptive 'Overall' = 74.92%; stored /100. Identity: July 2024 addition; gpt-4o-mini-2024-07-18 was its only snapshot.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "charxiv-rq",
    "score": 0.339,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Gemini 1.5 Flash', Reasoning 'Overall' = 33.90%; stored /100. Identity: July 2024 addition; gemini-1.5-flash-001 was the only Gemini 1.5 Flash GA version then.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "internvl2-26b",
    "bench": "charxiv-rq",
    "score": 0.334,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'InternVL Chat V2.0 26B', Reasoning 'Overall' = 33.40%; stored /100. Identity: OpenGVLab/InternVL2-26B (README news 07/14/2024).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "internvl2-26b",
    "bench": "charxiv-dq",
    "score": 0.624,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'InternVL Chat V2.0 26B', Descriptive 'Overall' = 62.40%; stored /100. Identity: OpenGVLab/InternVL2-26B (README news 07/14/2024).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "charxiv-rq",
    "score": 0.322,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Sonnet', Reasoning 'Overall' = 32.20%; stored /100. Identity: paper v1 names claude-3-sonnet-20240229.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "charxiv-dq",
    "score": 0.7365,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Sonnet', Descriptive 'Overall' = 73.65%; stored /100. Identity: paper v1 names claude-3-sonnet-20240229.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "charxiv-rq",
    "score": 0.318,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Haiku', Reasoning 'Overall' = 31.80%; stored /100. Identity: paper v1 names claude-3-haiku-20240307.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "charxiv-dq",
    "score": 0.6508,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Haiku', Descriptive 'Overall' = 65.08%; stored /100. Identity: paper v1 names claude-3-haiku-20240307.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "charxiv-rq",
    "score": 0.316,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Phi-3 Vision', Reasoning 'Overall' = 31.60%; stored /100. Identity: microsoft/Phi-3-vision-128k-instruct, the only Phi-3 Vision release.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "phi-3-vision-128k-instruct",
    "bench": "charxiv-dq",
    "score": 0.6048,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Phi-3 Vision', Descriptive 'Overall' = 60.48%; stored /100. Identity: microsoft/Phi-3-vision-128k-instruct, the only Phi-3 Vision release.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "minicpm-v-2-6",
    "bench": "charxiv-rq",
    "score": 0.31,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'MiniCPM-V2.6 (Upsize+CoT)', Reasoning 'Overall' = 31.00%; stored /100. Identity: the authors' 'Upsize+CoT' configuration (a second row without CoT is also published).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "minicpm-v-2-6",
    "bench": "charxiv-dq",
    "score": 0.5705,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'MiniCPM-V2.6 (Upsize+CoT)', Descriptive 'Overall' = 57.05%; stored /100. Identity: the authors' 'Upsize+CoT' configuration (a second row without CoT is also published).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "charxiv-rq",
    "score": 0.302,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Opus', Reasoning 'Overall' = 30.20%; stored /100. Identity: paper v1 names claude-3-opus-20240229.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "charxiv-dq",
    "score": 0.7155,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Claude 3 Opus', Descriptive 'Overall' = 71.55%; stored /100. Identity: paper v1 names claude-3-opus-20240229.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "charxiv-rq",
    "score": 0.266,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Reka Flash', Reasoning 'Overall' = 26.60%; stored /100. Identity: paper v1 names reka-flash-20240226.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "charxiv-dq",
    "score": 0.5645,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'Reka Flash', Descriptive 'Overall' = 56.45%; stored /100. Identity: paper v1 names reka-flash-20240226.",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "charxiv-rq",
    "score": 0.225,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'LLaVA 1.6 Yi 34B', Reasoning 'Overall' = 22.50%; stored /100. Identity: liuhaotian/llava-v1.6-34b (Yi-34B base).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "charxiv-dq",
    "score": 0.5105,
    "source": "https://github.com/princeton-nlp/CharXiv/blob/7ebe88f78dee387691551f071abcb2b9e1a8025b/README.md",
    "by": "independent",
    "date": "2024-07-23",
    "note": "CharXiv README leaderboard (princeton-nlp/CharXiv @7ebe88f, Apache-2.0): the benchmark authors' own evaluation on the validation set, row 'LLaVA 1.6 Yi 34B', Descriptive 'Overall' = 51.05%; stored /100. Identity: liuhaotian/llava-v1.6-34b (Yi-34B base).",
    "source_id": "charxiv-readme-apache"
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "winogrande",
    "score": 0.8453,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'Winogrande', column 'Nemotron-51B' = 84.53. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "winogrande",
    "score": 0.8508,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'Winogrande', column 'Llama-3.1-70B-Instruct' = 85.08. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "arc-challenge",
    "score": 0.692,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'ARC Challenge', column 'Nemotron-51B' = 69.2. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "arc-challenge",
    "score": 0.7039,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'ARC Challenge', column 'Llama-3.1-70B-Instruct' = 70.39. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "mmlu",
    "score": 0.802,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'MMLU', column 'Nemotron-51B' = 80.2. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "mmlu",
    "score": 0.8166,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'MMLU', column 'Llama-3.1-70B-Instruct' = 81.66. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "hellaswag",
    "score": 0.8558,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'HellaSwag', column 'Nemotron-51B' = 85.58. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "hellaswag",
    "score": 0.8644,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'HellaSwag', column 'Llama-3.1-70B-Instruct' = 86.44. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-nemotron-51b",
    "bench": "gsm8k",
    "score": 0.9143,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'GSM8K', column 'Nemotron-51B' = 91.43. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "gsm8k",
    "score": 0.9204,
    "source": "https://arxiv.org/html/2411.19146v5",
    "by": "lab",
    "date": "2025-06-03",
    "note": "Puzzle paper (NVIDIA, arXiv 2411.19146v5) Table 2 'Accuracy comparison of Nemotron-51B with Llama-3.1-70B-Instruct across several benchmarks', row 'GSM8K', column 'Llama-3.1-70B-Instruct' = 92.04. Both models appear in the one table with the authors' own evaluation (NVIDIA authors; the paper compares child and parent models it evaluated itself); shots/harness are not printed in the paper, so the note keeps the generic benchmark id. Non-chat prompt row (not the '* Chat' rows). Nemotron-51B = Llama-3.1-Nemotron-51B-Instruct (Sec. 'We showcase ... Llama-3.1-Nemotron-51B-Instruct (Nemotron-51B)').",
    "source_id": "paper-2411.19146v5-puzzle"
  },
  {
    "model": "eurus-2-7b-prime",
    "bench": "aime-2024",
    "score": 0.267,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'AIME 2024', column 'Eurus-2-7B-PRIME' = 26.7 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "eurus-2-7b-sft",
    "bench": "aime-2024",
    "score": 0.033,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'AIME 2024', column 'Eurus-2-7B-SFT' = 3.3 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "eurus-2-7b-prime",
    "bench": "math-500",
    "score": 0.792,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'MATH-500', column 'Eurus-2-7B-PRIME' = 79.2 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "eurus-2-7b-sft",
    "bench": "math-500",
    "score": 0.651,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'MATH-500', column 'Eurus-2-7B-SFT' = 65.1 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "eurus-2-7b-prime",
    "bench": "minerva-math",
    "score": 0.386,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'Minerva Math', column 'Eurus-2-7B-PRIME' = 38.6 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "eurus-2-7b-sft",
    "bench": "minerva-math",
    "score": 0.327,
    "source": "https://huggingface.co/PRIME-RL/Eurus-2-7B-PRIME/blob/2f47d199012a5d37d0dcedbeff1e2802a0ea443c/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-31",
    "note": "PRIME-RL Eurus-2-7B-PRIME model card (@2f47d19) '## Evaluation' table, row 'Minerva Math', column 'Eurus-2-7B-SFT' = 32.7 (percent, stored /100; the '(+x)' delta in the PRIME column ignored). Vendor's own two models (Eurus-2-7B-PRIME and its SFT initial model). Decoding settings and sample counts are not printed on the card. Other columns (Qwen2.5-Math-7B-Instruct, Llama-3.1-70B-Instruct, GPT-4o) are other labs' cells and are not recorded.",
    "source_id": "first-party-prime-rl-scores"
  },
  {
    "model": "t2v-turbo-vc2",
    "bench": "vbench-t2v-total",
    "score": 0.8101,
    "source": "https://arxiv.org/html/2405.18750v2",
    "by": "lab",
    "date": "2024-10-11",
    "note": "T2V-Turbo paper (arXiv 2405.18750v2) Table 1 'Automatic Evaluation on VBench', row 'Our T2V-Turbo (VC2)', column Total score = 81.01 (stored /100; VBench 1.0 T2V, 16 dimensions, 4-step inference per Sec. 4.1). The authors' own two distilled models (VC2 = distilled from VideoCrafter2, MS = from ModelScopeT2V). Baseline rows (ModelScopeT2V, VideoCrafter2, proprietary models) are copied leaderboard values, not authors' runs, and are not recorded.",
    "source_id": "paper-2405.18750v2-t2vturbo"
  },
  {
    "model": "t2v-turbo-ms",
    "bench": "vbench-t2v-total",
    "score": 0.8062,
    "source": "https://arxiv.org/html/2405.18750v2",
    "by": "lab",
    "date": "2024-10-11",
    "note": "T2V-Turbo paper (arXiv 2405.18750v2) Table 1 'Automatic Evaluation on VBench', row 'Our T2V-Turbo (MS)', column Total score = 80.62 (stored /100; VBench 1.0 T2V, 16 dimensions, 4-step inference per Sec. 4.1). The authors' own two distilled models (VC2 = distilled from VideoCrafter2, MS = from ModelScopeT2V). Baseline rows (ModelScopeT2V, VideoCrafter2, proprietary models) are copied leaderboard values, not authors' runs, and are not recorded.",
    "source_id": "paper-2405.18750v2-t2vturbo"
  },
  {
    "model": "bagel",
    "bench": "xtc-bench-gen-overall",
    "score": 0.725,
    "source": "https://arxiv.org/html/2604.25072v1",
    "by": "independent",
    "date": "2026-04-27",
    "note": "XTC-Bench paper (arXiv 2604.25072v1, 27 Apr 2026) Table 2 'Generation performance (GG) evaluated on scene graph nodes', row 'BAGEL-7B', column 'Overall Gen.' = 0.725 (continuous 0-1 scene-graph agreement over all nodes). Authors' own run: Sec. 4.1 'We evaluate eight open-sourced uMMs with XTC-Bench, including BAGEL 7B, BLIP3-o-8B, Janus-Pro-7B, MMaDA-8B, OmniGen-2, Show-o, Show-o2-7B, and Tar-7B'. Show-o (no resolution/checkpoint stated), MMaDA-8B, Tar-7B and the commercial GPT/Gemini rows are not recorded.",
    "source_id": "paper-2604.25072v1-xtcbench"
  },
  {
    "model": "blip3o-8b",
    "bench": "xtc-bench-gen-overall",
    "score": 0.545,
    "source": "https://arxiv.org/html/2604.25072v1",
    "by": "independent",
    "date": "2026-04-27",
    "note": "XTC-Bench paper (arXiv 2604.25072v1, 27 Apr 2026) Table 2 'Generation performance (GG) evaluated on scene graph nodes', row 'BLIP3-o-8B', column 'Overall Gen.' = 0.545 (continuous 0-1 scene-graph agreement over all nodes). Authors' own run: Sec. 4.1 'We evaluate eight open-sourced uMMs with XTC-Bench, including BAGEL 7B, BLIP3-o-8B, Janus-Pro-7B, MMaDA-8B, OmniGen-2, Show-o, Show-o2-7B, and Tar-7B'. Show-o (no resolution/checkpoint stated), MMaDA-8B, Tar-7B and the commercial GPT/Gemini rows are not recorded.",
    "source_id": "paper-2604.25072v1-xtcbench"
  },
  {
    "model": "janus-pro-7b",
    "bench": "xtc-bench-gen-overall",
    "score": 0.499,
    "source": "https://arxiv.org/html/2604.25072v1",
    "by": "independent",
    "date": "2026-04-27",
    "note": "XTC-Bench paper (arXiv 2604.25072v1, 27 Apr 2026) Table 2 'Generation performance (GG) evaluated on scene graph nodes', row 'Janus-Pro-7B', column 'Overall Gen.' = 0.499 (continuous 0-1 scene-graph agreement over all nodes). Authors' own run: Sec. 4.1 'We evaluate eight open-sourced uMMs with XTC-Bench, including BAGEL 7B, BLIP3-o-8B, Janus-Pro-7B, MMaDA-8B, OmniGen-2, Show-o, Show-o2-7B, and Tar-7B'. Show-o (no resolution/checkpoint stated), MMaDA-8B, Tar-7B and the commercial GPT/Gemini rows are not recorded.",
    "source_id": "paper-2604.25072v1-xtcbench"
  },
  {
    "model": "omnigen2",
    "bench": "xtc-bench-gen-overall",
    "score": 0.499,
    "source": "https://arxiv.org/html/2604.25072v1",
    "by": "independent",
    "date": "2026-04-27",
    "note": "XTC-Bench paper (arXiv 2604.25072v1, 27 Apr 2026) Table 2 'Generation performance (GG) evaluated on scene graph nodes', row 'OmniGen-2', column 'Overall Gen.' = 0.499 (continuous 0-1 scene-graph agreement over all nodes). Authors' own run: Sec. 4.1 'We evaluate eight open-sourced uMMs with XTC-Bench, including BAGEL 7B, BLIP3-o-8B, Janus-Pro-7B, MMaDA-8B, OmniGen-2, Show-o, Show-o2-7B, and Tar-7B'. Show-o (no resolution/checkpoint stated), MMaDA-8B, Tar-7B and the commercial GPT/Gemini rows are not recorded.",
    "source_id": "paper-2604.25072v1-xtcbench"
  },
  {
    "model": "show-o2-7b",
    "bench": "xtc-bench-gen-overall",
    "score": 0.465,
    "source": "https://arxiv.org/html/2604.25072v1",
    "by": "independent",
    "date": "2026-04-27",
    "note": "XTC-Bench paper (arXiv 2604.25072v1, 27 Apr 2026) Table 2 'Generation performance (GG) evaluated on scene graph nodes', row 'Show-o-2-7B', column 'Overall Gen.' = 0.465 (continuous 0-1 scene-graph agreement over all nodes). Authors' own run: Sec. 4.1 'We evaluate eight open-sourced uMMs with XTC-Bench, including BAGEL 7B, BLIP3-o-8B, Janus-Pro-7B, MMaDA-8B, OmniGen-2, Show-o, Show-o2-7B, and Tar-7B'. Show-o (no resolution/checkpoint stated), MMaDA-8B, Tar-7B and the commercial GPT/Gemini rows are not recorded.",
    "source_id": "paper-2604.25072v1-xtcbench"
  },
  {
    "model": "show-o2-7b",
    "bench": "geneval",
    "score": 0.76,
    "source": "https://arxiv.org/html/2506.15564v3#S4.T5",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Show-o2 paper (Show Lab, NUS; arXiv 2506.15564v3, 22 Sep 2025) Table 5 (GenEval), row 'Show-o2 (Ours)' 7B, column 'Overall' = 0.76. Maker's own figure for its own model (vendor-reported); 66M image-text training pairs, default sampling (CFG and steps as in the paper's ablation). Other methods' rows in the table are other labs' and are not recorded.",
    "source_id": "first-party-showlab-show-o2-paper"
  },
  {
    "model": "show-o2-1-5b",
    "bench": "geneval",
    "score": 0.73,
    "source": "https://arxiv.org/html/2506.15564v3#S4.T5",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Show-o2 paper (Show Lab, NUS; arXiv 2506.15564v3, 22 Sep 2025) Table 5 (GenEval), row 'Show-o2 (Ours)' 1.5B, column 'Overall' = 0.73. Maker's own figure for its own model (vendor-reported); 66M image-text training pairs, default sampling (CFG and steps as in the paper's ablation). Other methods' rows in the table are other labs' and are not recorded.",
    "source_id": "first-party-showlab-show-o2-paper"
  },
  {
    "model": "show-o2-7b",
    "bench": "dpg-bench",
    "score": 0.8614,
    "source": "https://arxiv.org/html/2506.15564v3#S4.T6",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Show-o2 paper (Show Lab, NUS; arXiv 2506.15564v3, 22 Sep 2025) Table 6 (DPG-Bench), row 'Show-o2 (Ours)' 7B, column 'Overall' = 86.14 (stored /100). Maker's own figure for its own model (vendor-reported); 66M image-text training pairs, default sampling (CFG and steps as in the paper's ablation). Other methods' rows in the table are other labs' and are not recorded.",
    "source_id": "first-party-showlab-show-o2-paper"
  },
  {
    "model": "show-o2-1-5b",
    "bench": "dpg-bench",
    "score": 0.8502,
    "source": "https://arxiv.org/html/2506.15564v3#S4.T6",
    "by": "lab",
    "date": "2025-09-22",
    "note": "Show-o2 paper (Show Lab, NUS; arXiv 2506.15564v3, 22 Sep 2025) Table 6 (DPG-Bench), row 'Show-o2 (Ours)' 1.5B, column 'Overall' = 85.02 (stored /100). Maker's own figure for its own model (vendor-reported); 66M image-text training pairs, default sampling (CFG and steps as in the paper's ablation). Other methods' rows in the table are other labs' and are not recorded.",
    "source_id": "first-party-showlab-show-o2-paper"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "llm-blender-mixinstruct-gpt-rank",
    "score": 3.9,
    "source": "https://arxiv.org/html/2306.02561v3",
    "by": "independent",
    "date": "2023-06-30",
    "note": "LLM-Blender paper (arXiv 2306.02561v3) Table 2 'Empirical results on MixInstruct', row 'Open Assistant', column GPT-Rank (lower is better) = 3.9: mean rank of the model's answer among the 11 candidate LLMs' answers per example, ranked by ChatGPT pairwise comparisons (MaxWins), 4,771 test examples. Table 2 caption: 'GPT-Rank are the most important metric'. Checkpoint identity from the MixInstruct dataset card (huggingface.co/datasets/llm-blender/mix-instruct, licence mit): OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5. Rank is relative to this 11-model pool. Other candidates (Vicuna 13B v1.1, Alpaca native 7B, Koala 7B, Baize 7B, MOSS, MPT-7B-Instruct) are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2306.02561v3-llmblender"
  },
  {
    "model": "chatglm-6b",
    "bench": "llm-blender-mixinstruct-gpt-rank",
    "score": 5.63,
    "source": "https://arxiv.org/html/2306.02561v3",
    "by": "independent",
    "date": "2023-06-30",
    "note": "LLM-Blender paper (arXiv 2306.02561v3) Table 2 'Empirical results on MixInstruct', row 'ChatGLM', column GPT-Rank (lower is better) = 5.63: mean rank of the model's answer among the 11 candidate LLMs' answers per example, ranked by ChatGPT pairwise comparisons (MaxWins), 4,771 test examples. Table 2 caption: 'GPT-Rank are the most important metric'. Checkpoint identity from the MixInstruct dataset card (huggingface.co/datasets/llm-blender/mix-instruct, licence mit): THUDM/chatglm-6b. Rank is relative to this 11-model pool. Other candidates (Vicuna 13B v1.1, Alpaca native 7B, Koala 7B, Baize 7B, MOSS, MPT-7B-Instruct) are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2306.02561v3-llmblender"
  },
  {
    "model": "dolly-v2-12b",
    "bench": "llm-blender-mixinstruct-gpt-rank",
    "score": 6.9,
    "source": "https://arxiv.org/html/2306.02561v3",
    "by": "independent",
    "date": "2023-06-30",
    "note": "LLM-Blender paper (arXiv 2306.02561v3) Table 2 'Empirical results on MixInstruct', row 'Dolly V2', column GPT-Rank (lower is better) = 6.9: mean rank of the model's answer among the 11 candidate LLMs' answers per example, ranked by ChatGPT pairwise comparisons (MaxWins), 4,771 test examples. Table 2 caption: 'GPT-Rank are the most important metric'. Checkpoint identity from the MixInstruct dataset card (huggingface.co/datasets/llm-blender/mix-instruct, licence mit): databricks/dolly-v2-12b. Rank is relative to this 11-model pool. Other candidates (Vicuna 13B v1.1, Alpaca native 7B, Koala 7B, Baize 7B, MOSS, MPT-7B-Instruct) are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2306.02561v3-llmblender"
  },
  {
    "model": "stablelm-tuned-alpha-7b",
    "bench": "llm-blender-mixinstruct-gpt-rank",
    "score": 8.71,
    "source": "https://arxiv.org/html/2306.02561v3",
    "by": "independent",
    "date": "2023-06-30",
    "note": "LLM-Blender paper (arXiv 2306.02561v3) Table 2 'Empirical results on MixInstruct', row 'StableLM', column GPT-Rank (lower is better) = 8.71: mean rank of the model's answer among the 11 candidate LLMs' answers per example, ranked by ChatGPT pairwise comparisons (MaxWins), 4,771 test examples. Table 2 caption: 'GPT-Rank are the most important metric'. Checkpoint identity from the MixInstruct dataset card (huggingface.co/datasets/llm-blender/mix-instruct, licence mit): stabilityai/stablelm-tuned-alpha-7b. Rank is relative to this 11-model pool. Other candidates (Vicuna 13B v1.1, Alpaca native 7B, Koala 7B, Baize 7B, MOSS, MPT-7B-Instruct) are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2306.02561v3-llmblender"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "llm-blender-mixinstruct-gpt-rank",
    "score": 8.81,
    "source": "https://arxiv.org/html/2306.02561v3",
    "by": "independent",
    "date": "2023-06-30",
    "note": "LLM-Blender paper (arXiv 2306.02561v3) Table 2 'Empirical results on MixInstruct', row 'Flan-T5', column GPT-Rank (lower is better) = 8.81: mean rank of the model's answer among the 11 candidate LLMs' answers per example, ranked by ChatGPT pairwise comparisons (MaxWins), 4,771 test examples. Table 2 caption: 'GPT-Rank are the most important metric'. Checkpoint identity from the MixInstruct dataset card (huggingface.co/datasets/llm-blender/mix-instruct, licence mit): google/flan-t5-xxl. Rank is relative to this 11-model pool. Other candidates (Vicuna 13B v1.1, Alpaca native 7B, Koala 7B, Baize 7B, MOSS, MPT-7B-Instruct) are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2306.02561v3-llmblender"
  },
  {
    "model": "aya-vision-32b",
    "bench": "vmmu-mean",
    "score": 0.2246,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Aya-Vision-32B', column Mean = 22.46 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "gemma-3-4b",
    "bench": "vmmu-mean",
    "score": 0.2545,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Gemma-3-4B', column Mean = 25.45 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "gemma-3-27b",
    "bench": "vmmu-mean",
    "score": 0.3909,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Gemma-3-27B', column Mean = 39.09 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "vmmu-mean",
    "score": 0.5213,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Qwen-2.5-VL-32B', column Mean = 52.13 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "vmmu-mean",
    "score": 0.5175,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Qwen-2.5-VL-72B', column Mean = 51.75 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "llama-4-scout",
    "bench": "vmmu-mean",
    "score": 0.4906,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Llama-4-Scout', column Mean = 49.06 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "llama-4-maverick",
    "bench": "vmmu-mean",
    "score": 0.3837,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Llama-4-Maverick', column Mean = 38.37 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "mistral-small-3-2",
    "bench": "vmmu-mean",
    "score": 0.3188,
    "source": "https://arxiv.org/html/2508.13680v5",
    "by": "independent",
    "date": "2026-09-23",
    "note": "VMMU paper (arXiv 2508.13680v5, 23 Sep 2026) Table 2 'Open-source VLMs', row 'Mistral-Small-3.2-24B', column Mean = 31.88 (percent, stored /100): mean accuracy over the seven VMMU task columns (Math, Phys, Chem, Bio, Geo, Drv, IQ), 2.5k Vietnamese multimodal multiple-choice questions. Authors' own run: Sec. 4 'We evaluate 3 groups of models on the full VMMU benchmark ... (1) 10 open-source VLMs'. Open-weight rows only; closed-source rows (API snapshots unstated) and models without a catalogue entry are not recorded.",
    "source_id": "paper-2508.13680v5-vmmu"
  },
  {
    "model": "ling-flash-2-0",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5174,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ling Flash 2.0' = 0.5174 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "intellect-3",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.566,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Intellect 3' = 0.566 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5021,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Olmo 3 32B Think' = 0.5021 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "olmo-3-1-32b-think",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5067,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Olmo 3.1 32B Think' = 0.5067 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "olmo-3-1-32b-instruct",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.456,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Olmo 3.1 32B Instruct' = 0.456 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.413,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Olmo 3 7B Think' = 0.413 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3503,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Olmo 3 7B Instruct' = 0.3503 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5364,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Llama 3.3 70B Instruct' = 0.5364 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3967,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Llama 3.1 8B Instruct' = 0.3967 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gemma-3-27b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.461,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Gemma 3 27B' = 0.461 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gemma-3-12b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4379,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Gemma 3 12B' = 0.4379 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3214,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Gemma 3 4B' = 0.3214 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5888,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Qwen3 Next 80B-A3B Thinking' = 0.5888 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5687,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Qwen3 Next 80B-A3B Instruct' = 0.5687 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "minimax-m2",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5708,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'MiniMax M2' = 0.5708 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "glm-4-5-air",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.541,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'GLM 4.5 Air' = 0.541 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5114,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Nemotron Nano V3 30B-A3B' = 0.5114 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4815,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Phi 4 Reasoning' = 0.4815 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5003,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Qwen2.5 32B Instruct' = 0.5003 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4452,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Granite 4.0H Small' = 0.4452 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "smollm3-3b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3548,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'SmolLM3 3B' = 0.3548 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-14b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.476,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 14B Instruct' = 0.476 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-8b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4442,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 8B Instruct' = 0.4442 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-3b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3927,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 3B Instruct' = 0.3927 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4807,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 14B Reasoning' = 0.4807 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.4312,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 8B Reasoning' = 0.4312 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "ministral-3-3b-reasoning",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.3741,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Ministral 3 3B Reasoning' = 0.3741 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5771,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'gpt-oss 120b (med)' = 0.5771 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.5198,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'gpt-oss 20b (med)' = 0.5198 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.6258,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Claude Sonnet 4.5' = 0.6258 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "medmarks-v-mean-win-rate",
    "score": 0.6628,
    "source": "https://arxiv.org/html/2605.01417v1",
    "by": "independent",
    "date": "2026-05-02",
    "note": "Medmarks paper (arXiv 2605.01417v1, 2 May 2026) Table 4 'All models by mean win rate on Medmarks-V', row 'Gemini 3 Pro Preview' = 0.6628 (weighted mean win rate over 19 verifiable medical benchmarks, against the other 70 configurations). Authors' own evaluation of 61 models / 71 configurations (open-weight models served with vLLM, API models via the Prime Intellect gateway, officially recommended sampling parameters; Sec. 2). Quantised (FP8/4-bit/8-bit) rows and reasoning-effort variants other than the default 'med' are not recorded; Qwen3 rows whose 2507 revision is not stated are not recorded.",
    "source_id": "paper-2605.01417v1"
  },
  {
    "model": "wan-2-1-1-3b",
    "bench": "vbench-t2v-total",
    "score": 0.8396,
    "source": "https://arxiv.org/html/2503.20314v2#S4.T7",
    "by": "lab",
    "date": "2025-04-19",
    "note": "Wan technical report (Alibaba Wan team; arXiv 2503.20314v2, 19 Apr 2025) VBench comparison table, row 'Wan 1.3B', column Total Score = 83.96%. Maker's own figure for its own model (vendor-reported; VBench 1.0 T2V total). Rows for other models in the table are other labs' leaderboard values and are not recorded.",
    "source_id": "first-party-alibaba-wan-paper"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "vbench-t2v-total",
    "score": 0.8622,
    "source": "https://arxiv.org/html/2503.20314v2#S4.T7",
    "by": "lab",
    "date": "2025-04-19",
    "note": "Wan technical report (Alibaba Wan team; arXiv 2503.20314v2, 19 Apr 2025) VBench comparison table, row 'Wan 14B (2025-02-24)', column Total Score = 86.22%. Maker's own figure for its own model (vendor-reported; VBench 1.0 T2V total). Rows for other models in the table are other labs' leaderboard values and are not recorded.",
    "source_id": "first-party-alibaba-wan-paper"
  },
  {
    "model": "cosmos-1-0-diffusion-7b-text2world",
    "bench": "cosmos-3d-sampson-error",
    "score": 0.355,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-7B-Text2World', column 'Sampson error' = 0.355. NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-diffusion-7b-text2world",
    "bench": "cosmos-3d-pose-success",
    "score": 0.626,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-7B-Text2World', column 'Pose estimation success rate (%)' = 62.6% (stored /100). NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-diffusion-7b-video2world",
    "bench": "cosmos-3d-sampson-error",
    "score": 0.473,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-7B-Video2World', column 'Sampson error' = 0.473. NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-diffusion-7b-video2world",
    "bench": "cosmos-3d-pose-success",
    "score": 0.684,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-7B-Video2World', column 'Pose estimation success rate (%)' = 68.4% (stored /100). NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-autoregressive-4b",
    "bench": "cosmos-3d-sampson-error",
    "score": 0.433,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-4B', column 'Sampson error' = 0.433. NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-autoregressive-4b",
    "bench": "cosmos-3d-pose-success",
    "score": 0.356,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-4B', column 'Pose estimation success rate (%)' = 35.6% (stored /100). NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-autoregressive-5b-video2world",
    "bench": "cosmos-3d-sampson-error",
    "score": 0.392,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-5B-Video2World', column 'Sampson error' = 0.392. NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "cosmos-1-0-autoregressive-5b-video2world",
    "bench": "cosmos-3d-pose-success",
    "score": 0.27,
    "source": "https://arxiv.org/html/2501.03575v3",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Cosmos World Foundation Model Platform paper (NVIDIA; arXiv 2501.03575v3) Table 19 'Evaluation of 3D consistency on base Cosmos models', row 'Cosmos-Predict1-5B-Video2World', column 'Pose estimation success rate (%)' = 27.0% (stored /100). NVIDIA's own models; the table also carries VideoLDM (another lab, not recorded) and a real-video reference row (not a model). Table 10 maps Cosmos-Predict1 names to the diffusion (7B/14B Text2World, Video2World) and autoregressive (4B/12B, 5B/13B Video2World) families.",
    "source_id": "paper-2501.03575v3-cosmos"
  },
  {
    "model": "aya-vision-32b",
    "bench": "kaleidoscope-overall",
    "score": 0.3927,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'Aya-Vision-32B', column Overall Acc. = 39.27 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "kaleidoscope-overall",
    "score": 0.5294,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'Qwen2.5-VL-72B', column Overall Acc. = 52.94 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "kaleidoscope-overall",
    "score": 0.4821,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'Qwen2.5-VL-32B', column Overall Acc. = 48.21 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "molmo-7b-d",
    "bench": "kaleidoscope-overall",
    "score": 0.3287,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'Molmo-7B-D', column Overall Acc. = 32.87 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "kaleidoscope-overall",
    "score": 0.5832,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'GPT-4o', column Overall Acc. = 58.32 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "kaleidoscope-overall",
    "score": 0.6291,
    "source": "https://arxiv.org/html/2504.07072v2",
    "by": "independent",
    "date": "2025-04-29",
    "note": "Kaleidoscope paper (arXiv 2504.07072v2) main results table, row 'Claude 3.5 Sonnet', column Overall Acc. = 62.91 (percent over all 20,911 questions, stored /100). Authors' own evaluation (Sec. 3.2): open-weight models run locally with temperature 0.7, 1024 max tokens, image size 512x512; closed models with chain-of-thought prompts; Sec. 3.1 names GPT-4o as 2024/08/06 and Claude 3.5 Sonnet as 2024/10/22. Overall Acc. is over all responses (format errors counted wrong), not the 'valid responses' column. Gemini 1.5 Pro (snapshot unstated), Aya-Vision-8B, Qwen2.5-VL-7B/3B and Pangea-7B are not recorded (not catalogue checkpoints or unstated).",
    "source_id": "paper-2504.07072v2-kaleidoscope"
  },
  {
    "model": "bloom-7b1",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.032,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'BLOOM-7.1B', zero-shot EM = 3.2 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "bloom-7b1",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.1815,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'BLOOM-7.1B', 10-shot EM = 18.15 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-7b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.1465,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-7B', zero-shot EM = 14.65 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-7b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.3505,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-7B', 10-shot EM = 35.05 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-13b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.2135,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-13B', zero-shot EM = 21.35 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-13b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.412,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-13B', 10-shot EM = 41.2 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-33b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.2725,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-33B', zero-shot EM = 27.25 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-33b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.489,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-33B', 10-shot EM = 48.9 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-65b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.3525,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-65B', zero-shot EM = 35.25 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "llama-65b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.5245,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'LLaMA-65B', 10-shot EM = 52.45 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.343,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Vicuna-33B-v1.3', zero-shot EM = 34.3 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.44,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Vicuna-33B-v1.3', 10-shot EM = 44.0 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.206,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'FLAN-T5-XXL (11B)', zero-shot EM = 20.6 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "flan-t5-xxl",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.2095,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'FLAN-T5-XXL (11B)', 10-shot EM = 20.95 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-7b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.236,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-7B', zero-shot EM = 23.6 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-7b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.3045,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-7B', 10-shot EM = 30.45 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-40b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.2655,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-40B', zero-shot EM = 26.55 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-40b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.4225,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-40B', 10-shot EM = 42.25 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-180b-chat",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.3995,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-180B-chat', zero-shot EM = 39.95 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "falcon-180b-chat",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.493,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Falcon-180B-chat', 10-shot EM = 49.3 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "mpt-7b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.0345,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'MPT-7B', zero-shot EM = 3.45 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "mpt-7b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.3045,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'MPT-7B', 10-shot EM = 30.45 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "pythia-6-9b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.11,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Pythia-6.9B', zero-shot EM = 11.0 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "pythia-6-9b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.217,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Pythia-6.9B', 10-shot EM = 21.7 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "pythia-12b",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.1525,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Pythia-12B', zero-shot EM = 15.25 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "pythia-12b",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.232,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Pythia-12B', 10-shot EM = 23.2 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "factual-recall-premium2k-em-0shot",
    "score": 0.2845,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Mistral-7B', zero-shot EM = 28.45 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, no exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "factual-recall-premium2k-em-10shot",
    "score": 0.4075,
    "source": "https://arxiv.org/html/2404.16164v1",
    "by": "independent",
    "date": "2024-04-24",
    "note": "'Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall' (arXiv 2404.16164v1) Table 1 'Benchmarking results on Premium2k', row 'Mistral-7B', 10-shot EM = 40.75 (percent exact match, stored /100). Authors' own run (Sec. 4: open models via Hugging Face, OpenAI APIs in July 2023); prompts per the paper's Tables 5-7 (instruction + question, 10 exemplars). Closed GPT rows, the instruct/chat variants other than Falcon-180B-chat, Vicuna 7B/13B v1.3, OpenLLaMA, T0++/UL2, MPT-30B and Falcon-180B base are not recorded (not catalogue checkpoints).",
    "source_id": "paper-2404.16164v1-factrecall"
  },
  {
    "model": "cosmos3-super-text2image",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.8908,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Cosmos3-Super-Text2Image', column CVTG-500L PNED = 89.08 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos3-super-text2image",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.4122,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Cosmos3-Super-Text2Image', column CVTG-102ch PNED = 41.22 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos3-super-text2image",
    "bench": "hpsv3-cosmos3",
    "score": 11.6,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Cosmos3-Super-Text2Image', column HPSv3 = 11.6. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "nano-banana-pro",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.7179,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Gemini 3 Pro Image', column CVTG-500L PNED = 71.79 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "nano-banana-pro",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.764,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Gemini 3 Pro Image', column CVTG-102ch PNED = 76.4 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "nano-banana-pro",
    "bench": "hpsv3-cosmos3",
    "score": 11.78,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Gemini 3 Pro Image', column HPSv3 = 11.78. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "flux-2-dev",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.8498,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'FLUX.2-dev', column CVTG-500L PNED = 84.98 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "flux-2-dev",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.6874,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'FLUX.2-dev', column CVTG-102ch PNED = 68.74 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "flux-2-dev",
    "bench": "hpsv3-cosmos3",
    "score": 11.38,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'FLUX.2-dev', column HPSv3 = 11.38. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "qwen-image-2512",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.9086,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Qwen-Image-2512', column CVTG-500L PNED = 90.86 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "qwen-image-2512",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.7126,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Qwen-Image-2512', column CVTG-102ch PNED = 71.26 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "qwen-image-2512",
    "bench": "hpsv3-cosmos3",
    "score": 11.03,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Qwen-Image-2512', column HPSv3 = 11.03. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.8768,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Hunyuan 3.0', column CVTG-500L PNED = 87.68 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.7131,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Hunyuan 3.0', column CVTG-102ch PNED = 71.31 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "hpsv3-cosmos3",
    "score": 11.93,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Hunyuan 3.0', column HPSv3 = 11.93. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "z-image-turbo",
    "bench": "cvtg-500l-pned-cosmos3",
    "score": 0.8695,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Z-Image-Turbo', column CVTG-500L PNED = 86.95 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-500L: 500 CVTG-2K prompts upsampled into long dense descriptions; PNED = per-region normalised edit-distance accuracy of OCR text. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "z-image-turbo",
    "bench": "cvtg-102ch-pned-cosmos3",
    "score": 0.7332,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Z-Image-Turbo', column CVTG-102ch PNED = 73.32 (percent, stored /100). Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). CVTG-102ch: 102 NVIDIA-written Chinese-character prompts; PNED as above. Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "z-image-turbo",
    "bench": "hpsv3-cosmos3",
    "score": 11.36,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T11",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA; arXiv 2606.02800v4) Table 11 'Text-to-Image benchmark results', row 'Z-Image-Turbo', column HPSv3 = 11.36. Same table and run as the already-imported UniGenBench 'All' column (NVIDIA's own evaluation of NVIDIA and comparison models). HPSv3 human-preference score of the generated images (raw units). Closed-model row (Gemini 3 Pro Image) marked with a dagger in the table.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.259,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Llama-3.2-3B-Instruct', column Overall = 25.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.057,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Llama-3.2-1B-Instruct', column Overall = 5.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.213,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-3B-Instruct', column Overall = 21.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.2,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-1.5B-Instruct', column Overall = 20.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.117,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'deepseek-coder-1.3b-instruct', column Overall = 11.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.022,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Qwen-1.5B', column Overall = 2.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-3b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.142,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Falcon3-3B-Instruct', column Overall = 14.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-1b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.034,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Falcon3-1B-Instruct', column Overall = 3.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.077,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Phi-3-mini-128k-instruct', column Overall = 7.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.204,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'granite-3.0-2b-instruct', column Overall = 20.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.038,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'EXAONE-3.5-2.4B-Instruct', column Overall = 3.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.262,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Llama-3.1-8B-Instruct', column Overall = 26.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-9b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.39,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'gemma-2-9b-it', column Overall = 39.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.41,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-Coder-7B-Instruct', column Overall = 41.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.357,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-7B-Instruct', column Overall = 35.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.363,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'deepseek-coder-7b-instruct-v1.5', column Overall = 36.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.067,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Qwen-7B', column Overall = 6.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.049,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Llama-8B', column Overall = 4.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-7b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.143,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Falcon3-7B-Instruct', column Overall = 14.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.324,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Yi-Coder-9B-Chat', column Overall = 32.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.307,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Yi-1.5-9B-Chat', column Overall = 30.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.318,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'granite-3.0-8b-instruct', column Overall = 31.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.466,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-Coder-14B-Instruct', column Overall = 46.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.368,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-14B-Instruct', column Overall = 36.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.27,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-Coder-V2-Lite-Instruct', column Overall = 27.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.15,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-V2-Lite-Chat', column Overall = 15.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.313,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Qwen-14B', column Overall = 31.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-10b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.223,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Falcon3-10B-Instruct', column Overall = 22.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.229,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'WizardLM-13B-V1.2', column Overall = 22.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.302,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Phi-3-medium-128k-instruct', column Overall = 30.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-4",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.371,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'phi-4', column Overall = 37.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.331,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Mistral-Nemo-Instruct-2407', column Overall = 33.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-27b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.464,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'gemma-2-27b-it', column Overall = 46.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.441,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-Coder-32B-Instruct', column Overall = 44.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.45,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-32B-Instruct', column Overall = 45.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwq-32b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.329,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'QwQ-32B', column Overall = 32.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.328,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'deepseek-coder-33b-instruct', column Overall = 32.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.306,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Qwen-32B', column Overall = 30.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.311,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Yi-1.5-34B-Chat', column Overall = 31.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-small-2409",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.456,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Mistral-Small-Instruct-2409', column Overall = 45.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.354,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'granite-34b-code-instruct-8k', column Overall = 35.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.334,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'internlm2_5-20b-chat', column Overall = 33.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.503,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Llama-3.1-70B-Instruct', column Overall = 50.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.329,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Llama-3.3-70B-Instruct', column Overall = 32.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.487,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'Qwen2.5-72B-Instruct', column Overall = 48.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.397,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'deepseek-llm-67b-chat', column Overall = 39.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.287,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'DeepSeek-R1-Distill-Llama-70B', column Overall = 28.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "codereviewqa-acr-exact-match",
    "score": 0.309,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 8 'Automated Code Refinement - Exact Match Rate (%)', row 'WizardLM-70B-V1.0', column Overall = 30.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.788,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Llama-3.2-3B-Instruct', column Overall = 78.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.761,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Llama-3.2-1B-Instruct', column Overall = 76.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.688,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-3B-Instruct', column Overall = 68.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.782,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-1.5B-Instruct', column Overall = 78.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.019,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'deepseek-coder-1.3b-instruct', column Overall = 1.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-1.5B', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-3b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.52,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Falcon3-3B-Instruct', column Overall = 52.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-1b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.558,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Falcon3-1B-Instruct', column Overall = 55.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.769,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Phi-3-mini-128k-instruct', column Overall = 76.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.69,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'granite-3.0-2b-instruct', column Overall = 69.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.653,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'EXAONE-3.5-2.4B-Instruct', column Overall = 65.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.718,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Llama-3.1-8B-Instruct', column Overall = 71.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-9b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.741,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'gemma-2-9b-it', column Overall = 74.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.786,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-Coder-7B-Instruct', column Overall = 78.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.759,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-7B-Instruct', column Overall = 75.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.581,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'deepseek-coder-7b-instruct-v1.5', column Overall = 58.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.01,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-7B', column Overall = 1.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.004,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-8B', column Overall = 0.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-7b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.732,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Falcon3-7B-Instruct', column Overall = 73.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.521,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Yi-Coder-9B-Chat', column Overall = 52.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.786,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Yi-1.5-9B-Chat', column Overall = 78.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.559,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'granite-3.0-8b-instruct', column Overall = 55.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.739,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-Coder-14B-Instruct', column Overall = 73.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.733,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-14B-Instruct', column Overall = 73.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.757,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-Coder-V2-Lite-Instruct', column Overall = 75.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.768,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-V2-Lite-Chat', column Overall = 76.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.727,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-14B', column Overall = 72.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-10b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.447,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Falcon3-10B-Instruct', column Overall = 44.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.651,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'WizardLM-13B-V1.2', column Overall = 65.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.767,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Phi-3-medium-128k-instruct', column Overall = 76.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-4",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.766,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'phi-4', column Overall = 76.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.758,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Mistral-Nemo-Instruct-2407', column Overall = 75.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-27b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.74,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'gemma-2-27b-it', column Overall = 74.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.791,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-Coder-32B-Instruct', column Overall = 79.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.71,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-32B-Instruct', column Overall = 71.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwq-32b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.609,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'QwQ-32B', column Overall = 60.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.698,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'deepseek-coder-33b-instruct', column Overall = 69.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.732,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-32B', column Overall = 73.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.661,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Yi-1.5-34B-Chat', column Overall = 66.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-small-2409",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.767,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Mistral-Small-Instruct-2409', column Overall = 76.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.723,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'granite-34b-code-instruct-8k', column Overall = 72.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.757,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'internlm2_5-20b-chat', column Overall = 75.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.684,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Llama-3.1-70B-Instruct', column Overall = 68.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.66,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Llama-3.3-70B-Instruct', column Overall = 66.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.798,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'Qwen2.5-72B-Instruct', column Overall = 79.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.791,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'deepseek-llm-67b-chat', column Overall = 79.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.612,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-70B', column Overall = 61.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "codereviewqa-ctr-invariant-accuracy",
    "score": 0.786,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 9 'Change Type Recognition - Proportion of Plurality Agreement and Invariant Accuracy (%)', row 'WizardLM-70B-V1.0', column Overall = 78.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.004,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.2-3B-Instruct', column Overall = 0.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.2-1B-Instruct', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.35,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-3B-Instruct', column Overall = 35.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.012,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-1.5B-Instruct', column Overall = 1.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-1.3b-instruct', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-1.5B', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-3b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.017,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-3B-Instruct', column Overall = 1.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-1b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-1B-Instruct', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.307,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Phi-3-mini-128k-instruct', column Overall = 30.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.01,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'granite-3.0-2b-instruct', column Overall = 1.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.008,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'EXAONE-3.5-2.4B-Instruct', column Overall = 0.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.298,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.1-8B-Instruct', column Overall = 29.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-9b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.52,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'gemma-2-9b-it', column Overall = 52.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.107,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-7B-Instruct', column Overall = 10.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.342,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-7B-Instruct', column Overall = 34.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.046,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-7b-instruct-v1.5', column Overall = 4.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.002,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-7B', column Overall = 0.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.006,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-8B', column Overall = 0.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-7b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.226,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-7B-Instruct', column Overall = 22.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.039,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Yi-Coder-9B-Chat', column Overall = 3.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.432,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Yi-1.5-9B-Chat', column Overall = 43.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.227,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'granite-3.0-8b-instruct', column Overall = 22.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.373,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-14B-Instruct', column Overall = 37.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.571,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-14B-Instruct', column Overall = 57.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.062,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-Coder-V2-Lite-Instruct', column Overall = 6.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.069,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-V2-Lite-Chat', column Overall = 6.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.532,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-14B', column Overall = 53.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-10b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.12,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-10B-Instruct', column Overall = 12.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.001,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'WizardLM-13B-V1.2', column Overall = 0.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.43,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Phi-3-medium-128k-instruct', column Overall = 43.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-4",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.448,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'phi-4', column Overall = 44.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.29,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Mistral-Nemo-Instruct-2407', column Overall = 29.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-27b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.587,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'gemma-2-27b-it', column Overall = 58.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.643,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-32B-Instruct', column Overall = 64.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.676,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-32B-Instruct', column Overall = 67.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwq-32b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.638,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'QwQ-32B', column Overall = 63.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.032,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-33b-instruct', column Overall = 3.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.646,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-32B', column Overall = 64.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.246,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Yi-1.5-34B-Chat', column Overall = 24.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-small-2409",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.316,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Mistral-Small-Instruct-2409', column Overall = 31.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.207,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'granite-34b-code-instruct-8k', column Overall = 20.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.553,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'internlm2_5-20b-chat', column Overall = 55.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.69,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.1-70B-Instruct', column Overall = 69.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.666,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.3-70B-Instruct', column Overall = 66.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.583,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-72B-Instruct', column Overall = 58.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.378,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'deepseek-llm-67b-chat', column Overall = 37.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.473,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-70B', column Overall = 47.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "codereviewqa-clh-invariant-accuracy",
    "score": 0.192,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 11 'Change Localisation (Hard) - ... Invariant Accuracy (%)', row 'WizardLM-70B-V1.0', column Overall = 19.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.076,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.2-3B-Instruct', column Overall = 7.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.008,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.2-1B-Instruct', column Overall = 0.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.452,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-3B-Instruct', column Overall = 45.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.378,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-1.5B-Instruct', column Overall = 37.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-1.3b-instruct', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-1.5B', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-3b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.195,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-3B-Instruct', column Overall = 19.5 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-1b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-1B-Instruct', column Overall = 0.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.459,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Phi-3-mini-128k-instruct', column Overall = 45.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-2b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.142,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'granite-3.0-2b-instruct', column Overall = 14.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.267,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'EXAONE-3.5-2.4B-Instruct', column Overall = 26.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.543,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.1-8B-Instruct', column Overall = 54.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-9b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.496,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'gemma-2-9b-it', column Overall = 49.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.552,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-7B-Instruct', column Overall = 55.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.516,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-7B-Instruct', column Overall = 51.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-7b-instruct-v1-5",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.114,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-7b-instruct-v1.5', column Overall = 11.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.001,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-7B', column Overall = 0.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.068,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-8B', column Overall = 6.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-7b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.353,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-7B-Instruct', column Overall = 35.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-coder-9b-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.33,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Yi-Coder-9B-Chat', column Overall = 33.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.589,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Yi-1.5-9B-Chat', column Overall = 58.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-3-0-8b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.253,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'granite-3.0-8b-instruct', column Overall = 25.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.562,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-14B-Instruct', column Overall = 56.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.819,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-14B-Instruct', column Overall = 81.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.128,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-Coder-V2-Lite-Instruct', column Overall = 12.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.205,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-V2-Lite-Chat', column Overall = 20.5 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.668,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-14B', column Overall = 66.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "falcon3-10b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.18,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Falcon3-10B-Instruct', column Overall = 18.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.046,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'WizardLM-13B-V1.2', column Overall = 4.6 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.508,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Phi-3-medium-128k-instruct', column Overall = 50.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "phi-4",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.775,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'phi-4', column Overall = 77.5 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.528,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Mistral-Nemo-Instruct-2407', column Overall = 52.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "gemma-2-27b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.657,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'gemma-2-27b-it', column Overall = 65.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.857,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-Coder-32B-Instruct', column Overall = 85.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.895,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-32B-Instruct', column Overall = 89.5 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwq-32b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.772,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'QwQ-32B', column Overall = 77.2 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.09,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'deepseek-coder-33b-instruct', column Overall = 9.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.764,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Qwen-32B', column Overall = 76.4 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.57,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Yi-1.5-34B-Chat', column Overall = 57.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "mistral-small-2409",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.601,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Mistral-Small-Instruct-2409', column Overall = 60.1 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.193,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'granite-34b-code-instruct-8k', column Overall = 19.3 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "internlm2-5-20b-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.707,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'internlm2_5-20b-chat', column Overall = 70.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.767,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.1-70B-Instruct', column Overall = 76.7 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.78,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Llama-3.3-70B-Instruct', column Overall = 78.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.909,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'Qwen2.5-72B-Instruct', column Overall = 90.9 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.47,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'deepseek-llm-67b-chat', column Overall = 47.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.688,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'DeepSeek-R1-Distill-Llama-70B', column Overall = 68.8 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "codereviewqa-sih-invariant-accuracy",
    "score": 0.49,
    "source": "https://arxiv.org/html/2503.16167v2",
    "by": "independent",
    "date": "2025-05-31",
    "note": "CodeReviewQA paper (arXiv 2503.16167v2, 31 May 2025) appendix Table 13 'Solution Identification (Hard) - ... Invariant Accuracy (%)', row 'WizardLM-70B-V1.0', column Overall = 49.0 (percent, stored /100). Authors' own zero-shot, greedy (temperature 0) run of 72 Hugging Face models with default vLLM (Appendix E-F); the Hugging Face repo is given in the paper's Table 7 and matches the catalogue repo exactly.",
    "source_id": "paper-2503.16167v2-codereviewqa"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "think-bench-accuracy",
    "score": 0.9425,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Claude-3-7-sonnet', column Accuracy = 94.25% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "think-bench-accuracy",
    "score": 0.6291,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Deepseek-r1-distill-qwen-1.5b', column Accuracy = 62.91% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "think-bench-accuracy",
    "score": 0.6851,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Deepseek-r1-distill-qwen-7b', column Accuracy = 68.51% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "think-bench-accuracy",
    "score": 0.7018,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Deepseek-r1-distill-qwen-14b', column Accuracy = 70.18% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "think-bench-accuracy",
    "score": 0.7593,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Deepseek-r1-distill-qwen-32b', column Accuracy = 75.93% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "think-bench-accuracy",
    "score": 0.9491,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Qwen3-235b-a22b', column Accuracy = 94.91% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "qwq-plus",
    "bench": "think-bench-accuracy",
    "score": 0.896,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Qwq-plus', column Accuracy = 89.6% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "think-bench-accuracy",
    "score": 0.9185,
    "source": "https://arxiv.org/html/2505.22113v1",
    "by": "independent",
    "date": "2025-05-28",
    "note": "THINK-Bench paper (arXiv 2505.22113v1, 28 May 2025) Table 2 'Evaluation of Nine Metrics on CoT and Efficiency in Think-Bench', row 'Grok-3-mini-beta', column Accuracy = 91.85% (stored /100). Authors' own evaluation of eleven reasoning models (Sec. 4.1); judge Claude 3.7 Sonnet. The paper notes the DeepSeek distilled models and ERNIE-X1-Turbo sometimes emit reasoning without a final answer, which lowers their accuracy. Rows for DeepSeek-R1 (revision unstated), GLM-Z1-Air and ERNIE-X1-Turbo-32K (not catalogued) are not recorded.",
    "source_id": "paper-2505.22113v1-thinkbench"
  },
  {
    "model": "bloom-7b1",
    "bench": "copa",
    "score": 0.532,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'bloom-7b', Accuracy = 0.532. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "bloom",
    "bench": "copa",
    "score": 0.576,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'bloom-176b', Accuracy = 0.576. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "falcon-7b",
    "bench": "copa",
    "score": 0.538,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'falcon-7b', Accuracy = 0.538. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "falcon-40b",
    "bench": "copa",
    "score": 0.844,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'falcon-40b', Accuracy = 0.844. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "llama-2-7b",
    "bench": "copa",
    "score": 0.826,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'llama2-7b', Accuracy = 0.826. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "llama-2-13b",
    "bench": "copa",
    "score": 0.85,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'llama2-13b', Accuracy = 0.85. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "llama-2-70b",
    "bench": "copa",
    "score": 0.976,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'llama2-70b', Accuracy = 0.976. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "mpt-7b",
    "bench": "copa",
    "score": 0.656,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'mpt-7b', Accuracy = 0.656. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "phi-2",
    "bench": "copa",
    "score": 0.902,
    "source": "https://arxiv.org/html/2410.14897v1",
    "by": "independent",
    "date": "2024-10-18",
    "note": "'From Test-Taking to Test-Making' (arXiv 2410.14897v1) Table 3 'Answering accuracy on COPA test set', row 'phi-2', Accuracy = 0.902. Authors' own run (Sec. 5): 4-shot prompt (Table 2 template, four items from the COPA development set), greedy decoding, Hugging Face inference endpoints; checkpoints named in the Sec. 4 footnotes (bigscience/bloom and bloom-7b1, tiiuae/falcon-7b and 40b, meta-llama/Llama-2-7b/13b/70b, mosaicml/mpt-7b, microsoft/phi-2). COPA test set (500 items). Mistral-7B (v0.3 base), MPT-30B are not catalogue checkpoints and are not recorded.",
    "source_id": "paper-2410.14897v1-gencopa"
  },
  {
    "model": "command-a-plus",
    "bench": "aime-2025",
    "score": 0.9,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'AIME 25', Command A+ '90%'. Footnote 1: 'AIME 2025 is evaluated on the official 2025 set of 30 questions, each question being repeated 10 times. The score is the pass@1 over this 300 sample dataset.' Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "aime-2025",
    "score": 0.57,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'AIME 25', Command A Reasoning '57%'. Footnote 1: 'AIME 2025 is evaluated on the official 2025 set of 30 questions, each question being repeated 10 times. The score is the pass@1 over this 300 sample dataset.' Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-plus",
    "bench": "ifbench-prompt-loose",
    "score": 0.74,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'IFBench', Command A+ '74%'. Footnote 1: 'IFBench is for single turn, loose evaluation mode and the score reported corresponds to prompt accuracy over 1470 samples corresponding to 294 unique prompts repeated five times.' Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "ifbench-prompt-loose",
    "score": 0.36,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'IFBench', Command A Reasoning '36%'. Footnote 1: 'IFBench is for single turn, loose evaluation mode and the score reported corresponds to prompt accuracy over 1470 samples corresponding to 294 unique prompts repeated five times.' Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-plus",
    "bench": "scicode-unspecified",
    "score": 0.38,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'SciCode', Command A+ '38%'. Footnote 1: 'Scicode evaluated using 65 test problems which includes 288 subproblems. We prompt each subproblem with the scientist-annotated background following the methodology of Artificial Analysis.' Main-problem vs subproblem scoring not stated, so the unspecified id (Cohere's own run, not an Artificial Analysis number). Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-reasoning-08-2025",
    "bench": "scicode-unspecified",
    "score": 0.3,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 3 ('Performance for Command A+ and Command A Reasoning on a range of popular open-source benchmarks'), bar labelled 'SciCode', Command A Reasoning '30%'. Footnote 1: 'Scicode evaluated using 65 test problems which includes 288 subproblems. We prompt each subproblem with the scientist-annotated background following the methodology of Artificial Analysis.' Main-problem vs subproblem scoring not stated, so the unspecified id (Cohere's own run, not an Artificial Analysis number). Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-plus",
    "bench": "charxiv-dq",
    "score": 0.88,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 5 ('Comparison of multimodal performance for Command A+ and Command A Vision ... We follow standard methodology for the given benchmarks'), bar labelled 'CharXiv (descriptive)', Command A+ '88%'. Split not printed; the same chart's CharXiv (reasoning) labels (53%/47%) match the body text's 52.7%/46.9% already recorded under charxiv-rq. Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "command-a-vision-07-2025",
    "bench": "charxiv-dq",
    "score": 0.82,
    "source": "https://cohere.com/blog/command-a-plus",
    "by": "lab",
    "date": "2026-05-20",
    "note": "Image 5 ('Comparison of multimodal performance for Command A+ and Command A Vision ... We follow standard methodology for the given benchmarks'), bar labelled 'CharXiv (descriptive)', Command A Vision '82%'. Split not printed; see the Command A+ row. Cohere 'Introducing Command A+' blog post (May 20, 2026), fetched 2026-10-09; value printed as a data label on the chart (not read from bar height). vendor-reported (rule b): Cohere's own figure for its own model, hand-recorded. Reasoning setting not stated (Command A+ and Command A Reasoning are reasoning models; default thinking assumed by the post).",
    "source_id": "first-party-cohere-scores"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "terminal-bench-4",
    "score": 0.373,
    "source": "https://openai.com/index/gpt-6-astra",
    "by": "lab",
    "date": "2026-09-03",
    "note": "OpenAI 'GPT-6 Astra: A new generation of intelligence' (Sep 3, 2026), Coding table, row 'Terminal-Bench 4.0', column 'GPT-5.6 Sol' = 37.3% (also in the text: 'compared with 37.3% for GPT-5.6 Sol'). Page note: 'Evaluation scores are the maximum at any effort. GPT evaluations were run in our research environment or via our API'; footnote 2: 'GPT-5.6 Sol refers to the version available in our API, ChatGPT Codex, and ChatGPT Work.' Harness not stated. Read from Internet Archive capture 20261004191257 (openai.com returns HTTP 403 to direct fetches). vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded 2026-10-09; the public leaderboard's maintainer run (37.27%, Codex) is recorded separately.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "charxiv-rq",
    "score": 0.56,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'Apriel-1.6-15B-Thinker' = 56.00. Setting: own model; reasoning mode is the model default (always thinks). Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "apriel-1-6-15b-thinker",
    "bench": "charxiv-dq",
    "score": 0.8985,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'Apriel-1.6-15B-Thinker' = 89.85. Setting: own model; reasoning mode is the model default (always thinks). Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "gpt-5",
    "bench": "charxiv-rq",
    "score": 0.715,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'GPT-5 (high)' = 71.50. Setting: reasoning effort high (label); the same table also has a GPT-5 (Minimal) column (82.45 / 52.80), not used per highest-setting rule; gpt-5 = gpt-5-2025-08-07, the only GPT-5 snapshot. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "gpt-5",
    "bench": "charxiv-dq",
    "score": 0.9125,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'GPT-5 (high)' = 91.25. Setting: reasoning effort high (label); the same table also has a GPT-5 (Minimal) column (82.45 / 52.80), not used per highest-setting rule; gpt-5 = gpt-5-2025-08-07, the only GPT-5 snapshot. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "glm-4-5v",
    "bench": "charxiv-rq",
    "score": 0.63,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'GLM-4.5V (Thinking)' = 63.00. Setting: thinking mode; GLM-4.5V is the single open-weights release zai-org/GLM-4.5V. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "glm-4-5v",
    "bench": "charxiv-dq",
    "score": 0.908,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'GLM-4.5V (Thinking)' = 90.80. Setting: thinking mode; GLM-4.5V is the single open-weights release zai-org/GLM-4.5V. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "charxiv-rq",
    "score": 0.565,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'Gemini 2.5 Flash (high)' = 56.50. Setting: thinking budget high (label). No date printed; mapped to the stable gemini-2.5-flash (GA June 2025): the same card labels the September preview explicitly as \"Gemini 2.5 Flash (Sep)\" in its text table, and the image table repeats the Apriel-1.5 paper (2025-10-01) numbers, which predate wide use of the 09-2025 preview. Identity judgement, see md. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "charxiv-dq",
    "score": 0.836,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'Gemini 2.5 Flash (high)' = 83.60. Setting: thinking budget high (label). No date printed; mapped to the stable gemini-2.5-flash (GA June 2025): the same card labels the September preview explicitly as \"Gemini 2.5 Flash (Sep)\" in its text table, and the image table repeats the Apriel-1.5 paper (2025-10-01) numbers, which predate wide use of the 09-2025 preview. Identity judgement, see md. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "charxiv-rq",
    "score": 0.709,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'Claude Sonnet 3.7 (Thinking)' = 70.90. Setting: extended thinking on (budget not stated); claude-3-7-sonnet-20250219 is the only 3.7 Sonnet snapshot. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "charxiv-dq",
    "score": 0.9327,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'Claude Sonnet 3.7 (Thinking)' = 93.27. Setting: extended thinking on (budget not stated); claude-3-7-sonnet-20250219 is the only 3.7 Sonnet snapshot. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "grok-4-fast",
    "bench": "charxiv-rq",
    "score": 0.335,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (reasoning)', column 'Grok 4 Fast (Thinking)' = 33.50. Setting: reasoning mode of grok-4-fast (xAI unified model; thinking = grok-4-fast-reasoning). Low but far above the 10.8% random baseline printed by CharXiv. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "grok-4-fast",
    "bench": "charxiv-dq",
    "score": 0.6815,
    "source": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md",
    "by": "independent",
    "date": "2025-12-22",
    "note": "ServiceNow-AI/Apriel-1.6-15b-Thinker model card @5810d5f (MIT; commit 2025-12-22), Evaluation section, image-benchmark table under the line 'For image benchmarks, we report evaluations obtained by https://github.com/open-compass/VLMEvalKit' (the card also states 'All other benchmarks were evaluated internally'; only AA-Index text benchmarks are third-party). Row 'CharXiv (descriptive)', column 'Grok 4 Fast (Thinking)' = 68.15. Setting: reasoning mode of grok-4-fast (xAI unified model; thinking = grok-4-fast-reasoning). Low but far above the 10.8% random baseline printed by CharXiv. Split not printed; VLMEvalKit's CharXiv tasks use the validation set (CharXiv_reasoning_val / CharXiv_descriptive_val). Same ServiceNow measurements as the Apriel-1.5 paper (arXiv 2510.01141v1, Table 4) for the overlapping models; only the three models absent from this card are taken from the paper, so nothing is recorded twice.",
    "source_id": "card-final-apriel-1-6-15b-thinker"
  },
  {
    "model": "llama-4-maverick",
    "bench": "charxiv-rq",
    "score": 0.55,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Llama 4 Maverick', column 'CharXiv R' = 55.00. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: Llama-4-Maverick-17B-128E-Instruct is the only Maverick chat release; non-reasoning model. Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "llama-4-maverick",
    "bench": "charxiv-dq",
    "score": 0.877,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Llama 4 Maverick', column 'CharXiv Des' = 87.70. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: Llama-4-Maverick-17B-128E-Instruct is the only Maverick chat release; non-reasoning model. Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "charxiv-rq",
    "score": 0.391,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Qwen 2.5 VL 32B-Instruct', column 'CharXiv R' = 39.10. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: Qwen/Qwen2.5-VL-32B-Instruct. Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "charxiv-dq",
    "score": 0.725,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Qwen 2.5 VL 32B-Instruct', column 'CharXiv Des' = 72.50. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: Qwen/Qwen2.5-VL-32B-Instruct. Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "magistral-small-2509",
    "bench": "charxiv-rq",
    "score": 0.529,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Magistral Small-2509', column 'CharXiv R' = 52.90. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: mistralai/Magistral-Small-2509 (Magistral Small 1.2, reasoning model with vision). Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "magistral-small-2509",
    "bench": "charxiv-dq",
    "score": 0.8227,
    "source": "https://arxiv.org/abs/2510.01141v1",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Radhakrishna et al. (ServiceNow), 'Apriel-1.5-15b-Thinker', arXiv 2510.01141v1 (CC BY 4.0), Table 4 (multimodal benchmarks), row 'Magistral Small-2509', column 'CharXiv Des' = 82.27. §5.2: 'We evaluate vision capabilities using the VLMEvalKit toolkit ... adopt consistent prompts and inference settings across models to ensure fair comparisons' (authors' own runs; the AA-sourced numbers in the paper are the text Table 3 only). Identity: mistralai/Magistral-Small-2509 (Magistral Small 1.2, reasoning model with vision). Split not printed; VLMEvalKit CharXiv = validation set. The other Table 4 models (GPT-5, GLM-4.5V, Gemini 2.5 Flash, Claude 3.7 Sonnet, Grok 4 Fast) are recorded from the Apriel-1.6 card, which reprints the same ServiceNow measurements, so they are not duplicated here; Kimi-VL-2506 and Apriel-1.5 are not catalogued.",
    "source_id": "paper-2510.01141v1-apriel-1-5"
  },
  {
    "model": "claude-fable-5",
    "bench": "charxiv-rq",
    "score": 0.866,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'anthropic/claude-fable-5' / Subset 'Reasoning' / Samples 1000 / Epochs 1 = '86.6% accuracy' (stderr 1.1). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (1000 reasoning questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: Anthropic API id claude-fable-5 = catalogue claude-fable-5; reasoning/thinking not set in the README (Inspect defaults, no extended-thinking budget stated).",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "claude-fable-5",
    "bench": "charxiv-dq",
    "score": 0.943,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'anthropic/claude-fable-5' / Subset 'Descriptive' / Samples 4000 / Epochs 1 = '94.3% accuracy' (stderr 0.4). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (4000 descriptive questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: Anthropic API id claude-fable-5 = catalogue claude-fable-5; reasoning/thinking not set in the README (Inspect defaults, no extended-thinking budget stated).",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "charxiv-rq",
    "score": 0.459,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'openai/gpt-4o' / Subset 'Reasoning' / Samples 1000 / Epochs 1 = '45.9% accuracy' (stderr 1.6). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (1000 reasoning questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: API alias gpt-4o; OpenAI's gpt-4o model page (developers.openai.com/api/docs/models/gpt-4o, observed 2026-09-28 per the catalogue entry) gives gpt-4o-2024-08-06 as the default snapshot, so the alias resolved to that snapshot at run time.",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "charxiv-dq",
    "score": 0.795,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'openai/gpt-4o' / Subset 'Descriptive' / Samples 4000 / Epochs 1 = '79.5% accuracy' (stderr 0.6). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (4000 descriptive questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: API alias gpt-4o; OpenAI's gpt-4o model page (developers.openai.com/api/docs/models/gpt-4o, observed 2026-09-28 per the catalogue entry) gives gpt-4o-2024-08-06 as the default snapshot, so the alias resolved to that snapshot at run time.",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "gpt-6-astra",
    "bench": "charxiv-rq",
    "score": 0.893,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'openai/gpt-6-astra' / Subset 'Reasoning' / Samples 1000 / Epochs 1 = '89.3% accuracy' (stderr 1.0). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (1000 reasoning questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: OpenAI API id gpt-6-astra = catalogue gpt-6-astra; reasoning effort not stated (API default).",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "gpt-6-astra",
    "bench": "charxiv-dq",
    "score": 0.962,
    "source": "https://github.com/meridianlabs-ai/inspect_charxiv/blob/317e4ff4335d7d4242d8e3c803898b7345973bd2/README.md",
    "by": "independent",
    "date": "2026-09-11",
    "note": "Meridian Labs inspect_charxiv README @317e4ff (MIT; the commit the UK AISI inspect_evals register pins), 'Evaluation Report' table, row 'openai/gpt-6-astra' / Subset 'Descriptive' / Samples 4000 / Epochs 1 = '96.2% accuracy' (stderr 0.3). Rows for Fable 5 and GPT-4o were added 2026-09-11 (commit 114bd95), after the 2026-09-08 grading fixes; GPT-6 Astra added 2026-09-11 (414ee24). Validation set (4000 descriptive questions = CharXiv val). Protocol deviations stated by the repo: one question per prompt (original batches per chart), apply_corrections=True (typo-corrected targets accepted as alternatives), grader openai/gpt-4o alias (original grader gpt-4o-2024-05-13). Identity: OpenAI API id gpt-6-astra = catalogue gpt-6-astra; reasoning effort not stated (API default).",
    "source_id": "inspect-charxiv-meridianlabs"
  },
  {
    "model": "gpt-4-1",
    "bench": "charxiv-dq",
    "score": 0.879,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4.1'; value printed as '87.9%'. Column 'GPT-4.1' is the gpt-4.1 API model released with this post (snapshot gpt-4.1-2025-04-14); no reasoning-effort setting (non-reasoning model). No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "charxiv-dq",
    "score": 0.884,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4.1 mini'; value printed as '88.4%'. Column 'GPT-4.1 mini' is the gpt-4.1-mini API model released with this post (snapshot gpt-4.1-mini-2025-04-14); no reasoning-effort setting. No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "charxiv-dq",
    "score": 0.739,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4.1 nano'; value printed as '73.9%'. Column 'GPT-4.1 nano' is the gpt-4.1-nano API model released with this post (only snapshot gpt-4.1-nano-2025-04-14); no reasoning-effort setting. No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "charxiv-dq",
    "score": 0.853,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4o (2024-11-20)'; value printed as '85.3%'. Column header names the dated snapshot 'GPT-4o (2024-11-20)' = gpt-4o-2024-11-20; no reasoning-effort setting. No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "charxiv-dq",
    "score": 0.766,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4o mini'; value printed as '76.6%'. Column 'GPT-4o mini': gpt-4o-mini has a single API snapshot, gpt-4o-mini-2024-07-18; no reasoning-effort setting. No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "charxiv-dq",
    "score": 0.889,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'OpenAI o1 (high)'; value printed as '88.9%'. Column 'OpenAI o1 (high)': reasoning effort high (the highest o1 setting); the reasoning_effort parameter exists only on the API o1 snapshot o1-2024-12-17 (not o1-preview). No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "charxiv-dq",
    "score": 0.9,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5); read from Internet Archive capture 20261006095914 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv-D' / column 'GPT-4.5'; value printed as '90.0%'. Column 'GPT-4.5': OpenAI's GPT-4.5 in the API is gpt-4.5-preview, whose only snapshot is gpt-4.5-preview-2025-02-27 (the same post announces its deprecation); no reasoning-effort setting. No tool use is stated for the Vision table (the post's chart caption: 'In CharXiv-Reasoning, a model answers questions about charts from scientific papers'). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-2",
    "bench": "charxiv-rq",
    "score": 0.821,
    "source": "https://openai.com/index/introducing-gpt-5-2",
    "by": "lab",
    "date": "2025-12-11",
    "note": "OpenAI 'Introducing GPT-5.2' (December 11, 2025), appendix 'Detailed benchmarks', table 'Vision' (columns GPT-5.2 Thinking | GPT-5.2 Pro | GPT-5.1 Thinking); read from Internet Archive capture 20261005030805 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv reasoning (no tools)' / column 'GPT-5.2 Thinking'; value printed as '82.1%'. Reasoning effort xhigh (table footnote: 'Models were run with maximum available reasoning effort in our API (xhigh for GPT-5.2 Thinking & Pro, and high for GPT-5.1 Thinking)'). Identity: the post says 'GPT-5.2 Thinking is available today in the Responses API and Chat Completions API as gpt-5.2'. Tool setting printed in the row label: no tools (the separate 'CharXiv reasoning (w/ Python)' row is not used). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-1",
    "bench": "charxiv-rq",
    "score": 0.67,
    "source": "https://openai.com/index/introducing-gpt-5-2",
    "by": "lab",
    "date": "2025-12-11",
    "note": "OpenAI 'Introducing GPT-5.2' (December 11, 2025), appendix 'Detailed benchmarks', table 'Vision' (columns GPT-5.2 Thinking | GPT-5.2 Pro | GPT-5.1 Thinking); read from Internet Archive capture 20261005030805 of the page (openai.com returns HTTP 403 to direct fetches). Row 'CharXiv reasoning (no tools)' / column 'GPT-5.1 Thinking'; value printed as '67.0%'. Reasoning effort high (table footnote: maximum available API effort, 'high for GPT-5.1 Thinking'). Identity: OpenAI's GPT-5.1 post (openai.com/index/gpt-5-1) says 'GPT-5.1 Thinking will be released as GPT-5.1 in the API' = gpt-5.1. Tool setting printed in the row label: no tools (the separate 'CharXiv reasoning (w/ Python)' row is not used). Split not stated; CharXiv authors' default is validation. vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "charxiv-rq",
    "score": 0.842,
    "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/#evaluation",
    "by": "lab",
    "date": "2026-05-19",
    "note": "Gemini 3.5 Flash model card (HTML, 'Published 19 May 2026'), Evaluation > Results table ('Results as of May, 2026'), row 'Multimodal / CharXiv Reasoning (Information synthesis from complex charts)', notes 'No tools', column 'Gemini 3.5 Flash' = 84.2%. Evaluation PDF (deepmind.google/models/evals-methodology/gemini-3-5-flash -> gemini_3-5_flash_model_evaluation.pdf): 'CharXiv Reasoning results for Gemini models and GPT 5.5 are self-computed'; pass@1, Gemini API model-id gemini-3.5-flash, default sampling; thinking level not stated (default). Split not stated in this PDF; Google's Gemini 3 Pro/3 Flash methodology PDFs state the 1,000 validation reasoning questions. Same value repeated in the 3.6 Flash card. vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "charxiv-rq",
    "score": 0.833,
    "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/#evaluation",
    "by": "lab",
    "date": "2026-05-19",
    "note": "Gemini 3.5 Flash model card (HTML, published 19 May 2026), Results table, row 'CharXiv Reasoning', notes 'No tools', column 'Gemini 3.1 Pro' = 83.3%. Google's own model: the evaluation PDF says CharXiv Reasoning results for Gemini models are self-computed (pass@1, default sampling). Thinking level for the 3.1 Pro column not stated. The Gemini 3.1 Pro's own card has no CharXiv row; the same 83.3% is printed in the 3.6 Flash card. Catalogue id gemini-3-1-pro = 'Gemini 3.1 Pro (Preview)' (the only 3.1 Pro). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "charxiv-rq",
    "score": 0.803,
    "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/#evaluation",
    "by": "lab",
    "date": "2026-05-19",
    "note": "Gemini 3.5 Flash model card (HTML, published 19 May 2026), Results table, row 'CharXiv Reasoning', notes 'No tools', column 'Gemini 3 Flash' = 80.3%; self-computed per the 3.5 Flash evaluation PDF ('CharXiv Reasoning results for Gemini models ... are self-computed'). Same 80.3% ('No tools', column 'Gemini 3 Flash Thinking') on the Gemini 3 Flash launch page deepmind.google/models/gemini/flash/ in Internet Archive captures 20260101235310-20260502123253; the Gemini 3 Flash eval PDF names API model-id gemini-3-flash-preview and the 1,000 validation reasoning questions. Gemini 3 Flash's own card/eval PDF tables are images (not used). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "charxiv-rq",
    "score": 0.732,
    "source": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite/#evaluation",
    "by": "lab",
    "date": "2026-03-03",
    "note": "Gemini 3.1 Flash-Lite model card (HTML, 'Published 3 March 2026'), Results table ('as of March, 2026'), row 'CharXiv Reasoning (Information synthesis from complex charts)', column 'Gemini 3.1 Flash-Lite High' = 73.2%. The row's Notes cell is blank, but GPT-5 mini's cell is marked '(+ python)', and the 3.5 Flash-Lite card prints the same 73.2% for 3.1 Flash-Lite under 'No tools' (with tools 75.6%), so this is the no-tools score. Evaluation PDF (gemini_3-1_flash-lite_model_evaluation.pdf): 'CharXiv Reasoning results for Gemini models ... are self-computed'; pass@1, Gemini API model-id gemini-3.1-flash-lite-preview, default sampling, high thinking. vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "charxiv-rq",
    "score": 0.637,
    "source": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite/#evaluation",
    "by": "lab",
    "date": "2026-03-03",
    "note": "Gemini 3.1 Flash-Lite model card (HTML, published 3 March 2026), Results table, row 'CharXiv Reasoning', column 'Gemini 2.5 Flash Dynamic' = 63.7% (dynamic thinking, the only setting printed). Self-computed: the evaluation PDF says 'CharXiv Reasoning results for Gemini models ... are self-computed'. The same 63.7% is printed as 'No tools', column 'Gemini 2.5 Flash Thinking', on the archived Gemini 3 Flash page (deepmind.google/models/gemini/flash/, IA capture 20260101235310). Mapped to the GA gemini-2-5-flash (the page names no snapshot; the GA model was the served 'Gemini 2.5 Flash' in 2026). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "charxiv-rq",
    "score": 0.555,
    "source": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite/#evaluation",
    "by": "lab",
    "date": "2026-03-03",
    "note": "Gemini 3.1 Flash-Lite model card (HTML, published 3 March 2026), Results table, row 'CharXiv Reasoning', column 'Gemini 2.5 Flash-Lite Dynamic' = 55.5% (dynamic thinking, the only setting printed; no tools, see the 3.1 Flash-Lite row note). Self-computed per the evaluation PDF ('CharXiv Reasoning results for Gemini models ... are self-computed'). Mapped to GA gemini-2-5-flash-lite (no snapshot named on the page). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-5-flash-lite",
    "bench": "charxiv-rq",
    "score": 0.745,
    "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/#evaluation",
    "by": "lab",
    "date": "2026-07-21",
    "note": "Gemini 3.5 Flash-Lite model card (HTML, 'Published 21 July 2026'), Results table ('as of July, 2026'), row 'CharXiv Reasoning (Information synthesis from complex charts)', notes 'No tools', column 'Gemini 3.5 Flash-Lite' = 74.5% (with tools 76.5% not used). Evaluation PDF (gemini_3-5_flash-lite_model_evaluation.pdf): 'CharXiv Reasoning results for all models are self-computed'; pass@1, Gemini API model-id gemini-3.5-flash-lite, high thinking, default sampling. Same cell on deepmind.google/models/gemini/flash-lite/. vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "charxiv-rq",
    "score": 0.814,
    "source": "https://deepmind.google/models/gemini/pro/",
    "by": "lab",
    "date": "2025-11-18",
    "note": "Gemini 3 Pro launch page deepmind.google/models/gemini/pro/ (Gemini 3 launched 18 Nov 2025), 'Performance' table, row 'Information synthesis from complex charts / CharXiv Reasoning', column 'Gemini 3 Pro' = 81.4%. Read from Internet Archive capture 20251203081720 (https://web.archive.org/web/20251203081720/https://deepmind.google/models/gemini/pro/); the live page now shows the Gemini 3.1 Pro table, which has no CharXiv row. Same 81.4% labelled 'No tools' (column 'Gemini 3 Pro Thinking') on the archived Gemini 3 Flash page (captures 20260101235310 to 20260502123253). Gemini 3 Pro eval PDF (gemini_3_pro_model_evaluation.pdf): pass@1, Gemini API model-id gemini-3-pro-preview, default sampling; 'CharXiV Reasoning results are on 1000 reasoning questions from the validation split'. The model card's own table is an image (not used). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "charxiv-rq",
    "score": 0.696,
    "source": "https://deepmind.google/models/gemini/pro/",
    "by": "lab",
    "date": "2025-11-18",
    "note": "Gemini 3 Pro launch page deepmind.google/models/gemini/pro/, 'Performance' table, row 'CharXiv Reasoning', column 'Gemini 2.5 Pro' = 69.6%. Read from Internet Archive capture 20251203081720; the live page no longer prints this row. Same 69.6% labelled 'No tools' (column 'Gemini 2.5 Pro Thinking') on the archived Gemini 3 Flash page (captures 20260101235310 to 20260502123253). Google's own Gemini model on Google's page. The Gemini 3 Pro eval PDF says Gemini results are run through the Gemini API on the 1,000 validation reasoning questions; the PDF says Google computed CharXiv itself where no self-reported number existed. Mapped to the GA gemini-2-5-pro (no snapshot named; Nov 2025). vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "charxiv-rq",
    "score": 0.813,
    "source": "https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf#page=219",
    "by": "lab",
    "date": "2026-05-28",
    "note": "Anthropic Claude Opus 4.8 System Card (May 28, 2026), §8.12.4 'CharXiv Reasoning' prose (PDF p.219): 'Claude Opus 4.7 scored 81.3% and 90.1% in the same settings' (no tools / Python tools); no-tools figure recorded. Settings: 1,000 validation questions, adaptive thinking at max effort, five runs. §8.12 says Opus 4.8 grading removes the student's thinking trace and 'we re-evaluated all prior models with the new grader'. This is the same grader as the existing Opus 4.8 (80.5%), Sonnet 5 and Sonnet 4.6 rows. Opus 4.7's own card prints 82.1% under the earlier grader; that value is under skipped. vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "charxiv-rq",
    "score": 0.691,
    "source": "https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf#page=191",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Anthropic Claude Opus 4.7 System Card (April 16, 2026), §8.1 Evaluation summary table (PDF p.191), row 'CharXiv Reasoning' / 'No tools', column 'Claude Opus 4.6' = 69.1%. Footnote 39: 'Reflects updated CharXiv evaluation settings introduced in the Claude Mythos Preview System Card (p 194); differs from values in the Claude Opus 4.6 system card'. That change replaced the Claude Sonnet 4 grader with Claude Sonnet 4.6. The same figure is in the prose of §8.9.2 (p.204): 'Claude Opus 4.6 scored 69.1% and 84.7% in the same settings'. Settings: 1,000 validation questions, adaptive thinking at max effort, five runs. The own-card 68.5% (old grader) is under skipped. vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "charxiv-rq",
    "score": 0.657,
    "source": "https://www-cdn.anthropic.com/6a5fa276ac68b9aeb0c8b6af5fa36326e0e166dd/Claude%20Opus%204.6%20System%20Card.pdf#page=36",
    "by": "lab",
    "date": "2026-02-05",
    "note": "Anthropic Claude Opus 4.6 System Card (dated 'February 2026'; Opus 4.6 announced Feb 5, 2026), §2.19.3 'CharXiv Reasoning' prose (PDF p.36): Opus 4.6 68.5% without tools and 77.4% with an image-cropping tool; 'In the same settings, Claude Opus 4.5 scored 65.7% and 68.7%, respectively'. No-tools 65.7% recorded. Settings: 1,000 validation questions, adaptive thinking, max effort, five runs. CAVEAT: graded with the earlier Claude Sonnet 4 grader, which the Mythos Preview card (p.194) replaced with Sonnet 4.6 because it depressed scores. For Opus 4.6 the change was 68.5% to 69.1% without tools. This is the only Opus 4.5 CharXiv figure Anthropic printed as text. Opus 4.5's own card has no CharXiv. vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "charxiv-rq",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-08-14",
    "note": "Pinned Qwen/Qwen3.8-27B model card (apache-2.0), VL Performance table, section 'General Multimodal Intelligence', row 'Scientific chart analysis / CharXiv (RQ)', column 'Qwen3.8-27B': cell 'Without CI 83.7 / With CI 90.2'; the without-code-interpreter value 83.7 is used, the With CI (tool) value is not. Card footnote: 'MathVision, BabyVision, and CharXiv (RQ): Where both settings are available, cells report “Without CI” and “With CI” separately ... A small number of incorrect ground-truth annotations in MathVision and CharXiv (RQ) were corrected following manual verification, and all reported scores on those benchmarks were computed using the corrected annotations.' So this is CharXiv RQ on Qwen's corrected key. Thinking mode (card default; no effort stated for the row). Split not stated; CharXiv authors' default is validation. Checkpoint pinned by repo sha 1d4bf0f. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "charxiv-rq",
    "score": 0.858,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-08-14",
    "note": "Pinned Qwen/Qwen3.8-27B model card (apache-2.0), VL Performance table, section 'General Multimodal Intelligence', row 'Scientific chart analysis / CharXiv (RQ)', column 'Qwen3.7-Plus' (same-lab API model, catalogue qwen3-7-plus): cell 'Without CI 85.8 / With CI 85.9'; 85.8 (no code interpreter) used. Same corrected-annotation footnote as the Qwen3.8-27B row, so this differs from the 84.4 Qwen printed in the Qwen3.7-Plus launch post (original key; recorded separately under first-party-qwen-scores). Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. Run: with qwen3-8-27b this makes a 2-model run for card-qwen--qwen3.8-27b (the card's Qwen3.6-27B cell, 78.4, repeats the existing card-qwen--qwen3.6-27b figure and is not re-recorded).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "charxiv-rq",
    "score": 0.846,
    "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/de4b8e4d43b917e7706784d8bb445c9af86a3540/README.md",
    "by": "lab",
    "date": "2026-08-27",
    "note": "Pinned Qwen/Qwen3.8-Flash-Next model card (sha de4b8e4; licence Qwen Community License 1.0, not open, so recorded under rule b), benchmark table, section 'General Multimodal Intelligence', row 'Scientific chart analysis / CharXiv (RQ)', column 'Qwen3.8-Flash-Next': 'Without CI 84.6 / With CI 90.6'; 84.6 (no code interpreter) used. Footnote 5: 'MathVision, CharXiv (RQ): scores are reported as \"without CI / with CI\"'. This card's footnote mentions corrected annotations for MathVision only, but its Qwen3.8-27B (83.7) and Qwen3.7-Plus (85.8) cells equal the Qwen3.8-27B card's corrected-key values, so treat it as the corrected key. Same 84.6 in the launch post https://qwen.ai/blog?id=qwen3.8-flash-next (2026-08-26). Thinking mode (card default). Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "charxiv-rq",
    "score": 0.884,
    "source": "https://qwen.ai/blog?id=qwen3.8",
    "by": "lab",
    "date": "2026-08-03",
    "note": "Qwen3.8-Max launch post 'Qwen3.8-Max: A New Bar for Coding and Cowork' (August 3, 2026), multimodal table, section 'Document & Office Intelligence', row 'CharXiv (RQ)', column 'Qwen3.8-Max': '88.4 / 93.5'; footnote 1: 'MathVision, BabyVision, CharXiv (RQ), and ZeroBench: Scores are reported as “without CI / with CI.” A small number of incorrect ground-truth annotations in MathVision and CharXiv (RQ) were corrected following manual verification.' 88.4 = without code interpreter, corrected key. Identity: the post says Qwen3.8-Max is '2.4T parameters (95B active), with open weights releasing next week' and uses API model 'qwen3.8-max' -> catalogue qwen3-8-2-4t-a95b (Qwen/Qwen3.8-2.4T-A95B, undated qwen3.8-max), not the later dated snapshot qwen3-8-max-0902. The post gives reasoning_effort 'xhigh (default)'; the effort for this row is not stated. Split not stated; CharXiv authors' default is validation. Page is client-rendered; the post HTML was read on 2026-10-09 from Qwen's own article endpoint (https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.8&type=qwen_ai), which the blog page itself calls. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "charxiv-rq",
    "score": 0.844,
    "source": "https://qwen.ai/blog?id=qwen3.7-plus",
    "by": "lab",
    "date": "2026-06-01",
    "note": "Qwen3.7-Plus launch post 'Qwen3.7-Plus: Multimodal Agent Intelligence' (2026-06-01), 'Multimodal Benchmarks' table, section 'Multimodal Reasoning', row 'CharXiv(RQ)', column 'Qwen3.7-Plus': '85.9 / 84.4'; footnote: '* BabyVision and CharXiv(RQ): Scores are reported as \"with CI / without CI\".' -> 84.4 is without code interpreter (85.9 with CI not used). No annotation-correction note in this post (Qwen's August tables print 85.8 without CI for Qwen3.7-Plus on a corrected key; that figure is recorded under card-qwen--qwen3.8-27b). Split not stated; CharXiv authors' default is validation. Page is client-rendered; the post HTML was read on 2026-10-09 from Qwen's own article endpoint (https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.7-plus&type=qwen_ai), which the blog page itself calls. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "charxiv-rq",
    "score": 0.815,
    "source": "https://qwen.ai/blog?id=qwen3.6",
    "by": "lab",
    "date": "2026-04-02",
    "note": "Qwen3.6-Plus launch post 'Qwen3.6-Plus: Towards Real World Agents' (2026-04-02), vision-language table, section 'Text Recognition and Document Understanding', row 'CharXiv(RQ)', column 'Qwen3.6-Plus' = 81.5 (single value; no tool variant printed). The same 81.5 is reprinted for Qwen3.6-Plus in the Qwen3.7-Plus post. The Alibaba Cloud community mirror of this post shows the tables only as images. Split not stated; CharXiv authors' default is validation. Page is client-rendered; the post HTML was read on 2026-10-09 from Qwen's own article endpoint (https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.6&type=qwen_ai), which the blog page itself calls. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "charxiv-rq",
    "score": 0.808,
    "source": "https://qwen.ai/blog?id=qwen3.5",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen3.5 launch post 'Qwen3.5: Towards Native Multimodal Agents' (2026-02-16), vision-language table, section 'Text Recognition and Document Understanding', row 'CharXiv(RQ)', column 'Qwen3.5-397B-A17B' = 80.8 (open-weight model; not the hosted Qwen3.5-Plus id). FLAG: same figure as the existing row from card-qwen--qwen3.5-397b-a17b, but that card run holds only this one model for charxiv-rq and is therefore withheld; recording the text copy in the Qwen vendor run lets the figure count once. Drop this row if the owner prefers never to repeat a figure across sources. Split not stated; CharXiv authors' default is validation. Page is client-rendered; the post HTML was read on 2026-10-09 from Qwen's own article endpoint (https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.5&type=qwen_ai), which the blog page itself calls. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "charxiv-rq",
    "score": 0.661,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (licence: arXiv non-exclusive distribution, not open -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, section 'Document Understanding', row 'CharXiv(RQ)', column 'Qwen3-VL 235B-A22B / thinking' = 66.1 (the table marks values sourced from other reports with *, none on Qwen columns). The v2 HTML rendering omits Tables 2-4, so the PDF page is cited (PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380). The HF cards for this model show results only as images (not used). Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "charxiv-rq",
    "score": 0.621,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 on PDF page 15, section 'Document Understanding', row 'CharXiv(RQ)', column 'Qwen3-VL 235B-A22B / instruct' = 62.1. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "charxiv-dq",
    "score": 0.905,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 on PDF page 15, section 'Document Understanding', row 'CharXiv(DQ)', column 'Qwen3-VL 235B-A22B / thinking' = 90.5. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "charxiv-dq",
    "score": 0.894,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 on PDF page 15, section 'Document Understanding', row 'CharXiv(DQ)', column 'Qwen3-VL 235B-A22B / instruct' = 89.4. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "charxiv-rq",
    "score": 0.497,
    "source": "https://arxiv.org/html/2502.13923v1#S3.T5",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (only version; licence arXiv non-exclusive distribution -> rule b), HTML Table 5 (#S3.T5) 'Performance of Qwen2.5-VL and other models on OCR, chart, and document understanding benchmarks', row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 72B': '49.7/ 87.4' -> RQ 49.7. Qwen2.5-VL 72B = the released Qwen2.5-VL-72B-Instruct (catalogue qwen2-5-vl-72b). The Qwen2.5-VL-32B/72B HF cards have no CharXiv row. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "charxiv-dq",
    "score": 0.874,
    "source": "https://arxiv.org/html/2502.13923v1#S3.T5",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b), HTML Table 5 (#S3.T5), row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 72B': '49.7/ 87.4' -> DQ 87.4. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "kimi-k2-5",
    "bench": "charxiv-rq",
    "score": 0.775,
    "source": "https://huggingface.co/moonshotai/Kimi-K2.5/blob/4d01dfe0332d63057c186e0b262165819efb6611/README.md",
    "by": "lab",
    "date": "2026-04-30",
    "note": "Pinned moonshotai/Kimi-K2.5 model card (sha 4d01dfe; README front-matter 'license: other / license_name: modified-mit'), evaluation table, section 'Image & Video', row 'CharXiv (RQ)', column 'Kimi K2.5 (Thinking)' = 77.5. Footnotes: 'We report results for Kimi K2.5 ... with thinking mode enabled'; 'temperature = 1.0, top-p = 0.95'; vision benchmarks 'Max-tokens = 64k, averaged over three runs (avg@3)'. No tool use for this row. Same 77.5 reprinted as the Kimi K2.5 column of the K2.6 card and in the K2.5 tech report (arXiv 2602.02276v2, CC BY-NC-ND 4.0, not used). Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Moonshot's own model; other labs' columns not used.",
    "source_id": "first-party-moonshot-scores"
  },
  {
    "model": "kimi-k2-6",
    "bench": "charxiv-rq",
    "score": 0.804,
    "source": "https://huggingface.co/moonshotai/Kimi-K2.6/blob/7eb5002f6aadc958aed6a9177b7ed26bb94011bb/README.md",
    "by": "lab",
    "date": "2026-05-19",
    "note": "Pinned moonshotai/Kimi-K2.6 model card (sha 7eb5002; Modified MIT), evaluation table, section 'Vision', row 'CharXiv (RQ)', column 'Kimi K2.6' = 80.4; the separate row 'CharXiv (RQ) (w/ python)' (86.7) is a tool setting and not used. Footnotes: 'results for Kimi K2.6 and Kimi K2.5 with thinking mode enabled'; 'temperature = 1.0, top-p = 1.0, and a context length of 262,144 tokens'; vision 'Max-tokens = 98,304, averaged over three runs (avg@3)'. Same 80.4 on kimi.com/blog/kimi-k2-6. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b); other labs' columns not used.",
    "source_id": "first-party-moonshot-scores"
  },
  {
    "model": "kimi-k3",
    "bench": "charxiv-rq",
    "score": 0.848,
    "source": "https://huggingface.co/moonshotai/Kimi-K3/blob/f831ab66814297da540d832a5235f8e904f29d06/README.md",
    "by": "lab",
    "date": "2026-09-02",
    "note": "Pinned moonshotai/Kimi-K3 model card (sha f831ab6; licence 'Kimi K3 License', not open: Model-as-a-Service commercial restriction -> rule b), evaluation table, section 'Vision', row 'CharXiv (RQ)', column 'Kimi K3 (max)': '84.8 / 91.3'. Footnote: 'All Kimi K3 results are obtained with reasoning effort set to 'max' and temperature = 1.0 ... vision benchmarks without tools, we set top-p = 0.95 ... For HLE-Full, MMMU-Pro, CharXiv (RQ), MathVision, and ZeroBench, each cell reports the scores without and with tool augmentation (general tools for HLE-Full, Python for the vision benchmarks), in that order.' -> 84.8 without tools; 91.3 (Python) not used. Same figures on kimi.com/blog/kimi-k3. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b); other labs' columns not used.",
    "source_id": "first-party-moonshot-scores"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "charxiv-rq",
    "score": 0.801,
    "source": "https://mimo.xiaomi.com/mimo-v2-omni",
    "by": "lab",
    "date": "2026-03-18",
    "note": "Xiaomi MiMo-V2-Omni launch page (dated 'March 18th, 2026'), 'Perception' benchmark chart, entry 'CharXiv RQ' (subtitle 'Chart Understanding'), MiMo-V2-Omni = 80.1. FLAG: the number is in the page HTML as chart data (perceptionData: {name: 'CharXiv RQ', ... scores: {omni: 80.1, ...}}) and the page renders it as a text label (score.toFixed(1)) on the bar, so it is printed text, not a bar-height reading; the caller may still prefer to treat chart labels as out of scope. The prose says MiMo-V2-Omni was evaluated on image understanding '(MMMU-Pro, CharXiv RQ)'. Settings not stated. Same 80.1 is repeated for MiMo-V2-Omni on the MiMo-V2.5 page. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b): Xiaomi's own model; other labs' scores not used.",
    "source_id": "first-party-xiaomi-scores"
  },
  {
    "model": "mimo-v2-5",
    "bench": "charxiv-rq",
    "score": 0.81,
    "source": "https://mimo.xiaomi.com/mimo-v2-5",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Xiaomi MiMo-V2.5 launch page (dated 'April 22nd, 2026'), multimodal benchmark chart, entry 'CharXiv RQ' (subtitle 'Chart Understanding'), MiMo-V2.5 = 81.0, held in the page HTML as chart data (multimodalData scores {mimo25: 81.0, omni: 80.1, ...}) and rendered as a text label (score.toFixed(1)). FLAG: same figure as the existing card2-xiaomimimo-mimo-v2-5 row, which was read from a chart image in the card and is a one-model run (withheld); this text copy sits in Xiaomi's vendor run with mimo-v2-omni so it can count. Drop if the owner prefers not to repeat a figure. Settings not stated. Split not stated; CharXiv authors' default is validation. Vendor-reported (rule b).",
    "source_id": "first-party-xiaomi-scores"
  },
  {
    "model": "nova-premier",
    "bench": "charxiv-dq",
    "score": 0.846,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (published April 30, 2025 per the report), Table 1 (PDF page 3), row 'CharXiv (Descriptive/Reasoning)', column 'Nova Premier' printed '84.6/48.8'; descriptive value (first of the printed pair) used. Methods text: 'For chart understanding, we use CharXiv [24] as the benchmark without CoT prompt, and we report accuracy based on correctness binary score that is assigned by GPT-4o.' Split not stated; CharXiv authors' default is validation. Companion to the existing charxiv-rq row (48.8) from the same table. Vendor-reported (rule b): Amazon's own figure for its own model, hand-recorded; Claude-3.5 Sonnet-v2, Claude-3.7 Sonnet and GPT-4.5 columns not used.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "nova-pro",
    "bench": "charxiv-dq",
    "score": 0.705,
    "source": "https://cdn.amazon.science/e5/e6/ccc5378c42dca467d1abe1628ec9/amazon-nova-premier-technical-report-and-model-card.pdf",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Amazon 'Amazon Nova Premier: Technical Report and Model Card' (April 30, 2025), Table 1 (PDF page 3), row 'CharXiv (Descriptive/Reasoning)', column 'Nova Pro' printed '70.5/40.6'; descriptive value used. No CoT, GPT-4o-graded binary correctness (methods text). Split not stated; CharXiv authors' default is validation. Nova Pro here is the original Nova Pro (the report's comparator; Nova Premier is described as an addendum to the Nova family report), same mapping as the existing nova-pro charxiv-rq row (40.6). Vendor-reported (rule b), hand-recorded; other labs' columns not used.",
    "source_id": "first-party-amazon-scores"
  },
  {
    "model": "ministral-3-3b",
    "bench": "charxiv-dq",
    "score": 0.766,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct' (size row '3.8B'), cell '0.766'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0), section 'Chart / Document / OCR', row 'CharXiv_DQ' (printed 'CharXiv<sub>DQ</sub>'). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run with their vLLM implementation where available, else Hugging Face inference, 'following the authors' recommended vision and text inference settings', and 'Because the comparison focuses on instruction-tuned, non-reasoning models, we capped generation at 1,024 tokens'. Split not printed in the card; VLMEvalKit's only CharXiv descriptive dataset is the validation set (github.com/open-compass/VLMEvalKit/blob/c09a0585e4801065b322147774946099bc0fc85f/vlmeval/dataset/charxiv.py (DATASET_URL lists only CharXiv_descriptive_val / CharXiv_reasoning_val)). Values printed as fractions (no conversion). CharXiv row identical since the first card upload of 2026-08-10 (commit 4b0ecf6) through the pinned revision. Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "charxiv-dq",
    "score": 0.637,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct' (size row '4.2B', licence row 'MIT'), cell '0.637'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0), section 'Chart / Document / OCR', row 'CharXiv_DQ' (printed 'CharXiv<sub>DQ</sub>'). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run with their vLLM implementation where available, else Hugging Face inference, 'following the authors' recommended vision and text inference settings', and 'Because the comparison focuses on instruction-tuned, non-reasoning models, we capped generation at 1,024 tokens'. Split not printed in the card; VLMEvalKit's only CharXiv descriptive dataset is the validation set (github.com/open-compass/VLMEvalKit/blob/c09a0585e4801065b322147774946099bc0fc85f/vlmeval/dataset/charxiv.py (DATASET_URL lists only CharXiv_descriptive_val / CharXiv_reasoning_val)). Values printed as fractions (no conversion). CharXiv row identical since the first card upload of 2026-08-10 (commit 4b0ecf6) through the pinned revision. Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "charxiv-dq",
    "score": 0.751,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it' (size row '5.1B'), cell '0.751'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0), section 'Chart / Document / OCR', row 'CharXiv_DQ' (printed 'CharXiv<sub>DQ</sub>'). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run with their vLLM implementation where available, else Hugging Face inference, 'following the authors' recommended vision and text inference settings', and 'Because the comparison focuses on instruction-tuned, non-reasoning models, we capped generation at 1,024 tokens'. Split not printed in the card; VLMEvalKit's only CharXiv descriptive dataset is the validation set (github.com/open-compass/VLMEvalKit/blob/c09a0585e4801065b322147774946099bc0fc85f/vlmeval/dataset/charxiv.py (DATASET_URL lists only CharXiv_descriptive_val / CharXiv_reasoning_val)). Values printed as fractions (no conversion). CharXiv row identical since the first card upload of 2026-08-10 (commit 4b0ecf6) through the pinned revision. Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "charxiv-dq",
    "score": 0.761,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct' (size row '2.1B'), cell '0.761'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo (HF API returns 401/not found); the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b, model_repo Qwen/Qwen3.5-2B; Qwen-Base is separate). 'Instruct' read as the non-thinking use of Qwen/Qwen3.5-2B, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap. Card size 2.1B vs catalogue 2.27B (card sizes appear rounded/differently counted). Existing qwen3-5-2b CharXiv rows from Qwen's own card are thinking mode; keep the mode distinction. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0), section 'Chart / Document / OCR', row 'CharXiv_DQ' (printed 'CharXiv<sub>DQ</sub>'). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run with their vLLM implementation where available, else Hugging Face inference, 'following the authors' recommended vision and text inference settings', and 'Because the comparison focuses on instruction-tuned, non-reasoning models, we capped generation at 1,024 tokens'. Split not printed in the card; VLMEvalKit's only CharXiv descriptive dataset is the validation set (github.com/open-compass/VLMEvalKit/blob/c09a0585e4801065b322147774946099bc0fc85f/vlmeval/dataset/charxiv.py (DATASET_URL lists only CharXiv_descriptive_val / CharXiv_reasoning_val)). Values printed as fractions (no conversion). CharXiv row identical since the first card upload of 2026-08-10 (commit 4b0ecf6) through the pinned revision. Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-31b",
    "bench": "charxiv-rq",
    "score": 0.777,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab, pinned Muse Glimmer-30B model card (Apache-2.0), Benchmarks table, section '*Multimodal*', row 'Charxiv Reasoning', column 'Gemma4-31B Thinking Mode', cell '77.7'. Meta's linked 'Muse Glimmer Eval Methodology' (research.meta.ai/static/muse-glimmer-methodology, fetched 2026-10-09; settings only): 'CharXiv Reasoning: We use 1,000 chart reasoning questions from the CharXiv validation set. Grading uses gpt-oss-120b as an LLM judge ... We report the average task success rate across four attempts. We report internal run results for Gemma4-31B and our model, and a self-reported number for Qwen3.6-27B.' and 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64'. No tool use described for CharXiv (the same document names a Python cropping tool only for ScreenSpot-Pro). Validation split, no tools. Identity: Gemma4-31B = google/gemma-4-31B-it (catalogue gemma-4-31b model_repo). Rule (a): Apache-licensed card, Meta's own measurement of Google's model -> by: independent; joins the existing muse-glimmer-30b charxiv-rq row (78.8) from the same card, making a 2-model run. Qwen3.6-27B column (78.4) not used: methodology says it is a self-reported number (equals Qwen's own card value already recorded). Ledger speed-meta-muse-glimmer-30b already matches this exact URL; its name/attribution still say 'speed table / speed cells' and may want widening to benchmark cells.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "dots-tts-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0134,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2 'Zero-shot results on Seed-TTS-Eval', row 'dots.tts (Pretrain)' (2B), column test-en WER(%) = 1.34. Authors' own model: §3.3.1 'All numbers reported under the dots.tts (Pretrain) row are produced with the pretrained checkpoint from Section 3.2.2'; 'dots.tts-Pretrain and dots.tts-SOAR use 10 Euler solver steps with CFG ... effective NFE of 20 ... CFG scale is γ=1.2 ... All runs use float32 and have torch.compile disabled'. ASR: 'Whisper-Large-v3 for English'. Checkpoint: dots-studio/dots.tts-base @ b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1, whose card calls it 'the end-to-end pretrained checkpoint' / 'Pretrain (~1.5M h)' and prints the same Seed-TTS-Eval row (1.34 / 76.8).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0096,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2 'Zero-shot results on Seed-TTS-Eval', row 'dots.tts (Pretrain)' (2B), column test-zh WER = 0.96 (§3.3.1: 'CER reported as WER for Chinese for table compatibility'; 'Paraformer for Mandarin Chinese'). Same settings as SOAR: 10 Euler steps with CFG γ=1.2 (effective NFE 20), float32, torch.compile disabled. Checkpoint: dots-studio/dots.tts-base @ b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1 (card: 'Pretrain (~1.5M h)', same 0.96 / 80.5 test-zh cell).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.768,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2, row 'dots.tts (Pretrain)', column test-en SIM = 76.8 (percent; divided by 100). §3.3.1: 'speaker similarity (SIM) is the cosine similarity between WavLM-SV embeddings of the generated speech and the corresponding reference prompt'; §3.3.3: 'scored with the benchmark's reference ASR and WavLM-SV similarity' (prompt SIM). Settings: 10 Euler steps, CFG 1.2, float32. Checkpoint: dots-studio/dots.tts-base @ b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.805,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2, row 'dots.tts (Pretrain)', column test-zh SIM = 80.5 (percent; divided by 100). WavLM-SV cosine to the reference prompt (§3.3.1). Settings: 10 Euler steps, CFG 1.2, float32. Checkpoint: dots-studio/dots.tts-base @ b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.771,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Same table row as the existing dots-tts-soar WER result: Table 2, row 'dots.tts (SOAR)' (2B), column test-en SIM = 77.1 (percent; divided by 100). WavLM-SV cosine to the reference prompt (§3.3.1). Own measurement: float32, 10 Euler solver steps, CFG 1.2, effective NFE 20, torch.compile disabled. Added under brief step 5 (other existing benchmark of the same table) so the SIM run has both dots.tts checkpoints.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.81,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Table 2, row 'dots.tts (SOAR)' (2B), column test-zh SIM = 81.0 (percent; divided by 100). WavLM-SV cosine to the reference prompt (§3.3.1). Own measurement: float32, 10 Euler solver steps, CFG 1.2, effective NFE 20, torch.compile disabled. Added under brief step 5 (same table, existing benchmark id).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "hidream-i1-full",
    "bench": "dpg-bench",
    "score": 0.8589,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "README 'Evaluation' section, 'DPG-Bench — dense prompt alignment' table, row 'HiDream-I1-Full' (#Params '13.5B + 17B'), column Overall = 85.89 (divided by 100). Same table as the existing HiDream-O1-Image row (89.83). HiDream-I1-Full is HiDream.ai's own earlier model, not marked as copied, so it is treated as a lab measurement per the brief's same-family rule (the README states only 'We benchmark HiDream-O1-Image against state-of-the-art open-source and proprietary models on five widely-used evaluation suites'; the same 85.89 appears for HiDream-I1-Full in LongCat-Image Table 3, consistent with it being HiDream's own earlier published figure). Checkpoint: HiDream-I1-Full (17B), catalogue id hidream-i1-full.",
    "source_id": "visual-hidream"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.0132,
    "source": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/fd4b254389122332181a7c3db7f27e918eec64e3/README.md",
    "by": "lab",
    "date": "2026-01-23",
    "note": "Qwen3-TTS-12Hz-1.7B-Base card @ fd4b254, 'Evaluation' > 'Speech Generation Benchmarks', table 'Zero-shot speech generation on the Seed-TTS test set. Performance is measured by Word Error Rate (WER, ↓)', row 'Qwen3-TTS-12Hz-0.6B-Base', column test-en = 1.32 (percent; divided by 100). Qwen team's own run of its own smaller checkpoint in the same table as the existing 1.7B-Base row: 'During evaluation, we ran inference for all models with dtype=torch.bfloat16 and set max_new_tokens=2048. All other sampling parameters used the defaults from the checkpoint's generate_config.json. For the Seed-Test ... we set language=\"auto\"'. ASR transcriber not stated (hence the -asr-unstated id). Checkpoint: Qwen/Qwen3-TTS-12Hz-0.6B-Base (catalogue qwen3-tts-12hz-0-6b-base).",
    "source_id": "card2-qwen-qwen3-tts-12hz-1-7b-base"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-zh-err-asr-unstated",
    "score": 0.0092,
    "source": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/fd4b254389122332181a7c3db7f27e918eec64e3/README.md",
    "by": "lab",
    "date": "2026-01-23",
    "note": "Same card table ('Zero-shot speech generation on the Seed-TTS test set ... Word Error Rate (WER, ↓)'), row 'Qwen3-TTS-12Hz-0.6B-Base', column test-zh = 0.92 (percent, labelled WER; divided by 100). Settings: bfloat16, max_new_tokens=2048, checkpoint default sampling, language='auto'. ASR transcriber not stated. Checkpoint: Qwen/Qwen3-TTS-12Hz-0.6B-Base.",
    "source_id": "card2-qwen-qwen3-tts-12hz-1-7b-base"
  },
  {
    "model": "distil-large-v3",
    "bench": "ami-wer",
    "score": 0.1516,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'AMI', column 'distil-v3' = 15.16 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "gigaspeech-wer",
    "score": 0.1008,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'Gigaspeech', column 'distil-v3' = 10.08 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0254,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'LS Clean', column 'distil-v3' = 2.54 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0519,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'LS Other', column 'distil-v3' = 5.19 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "tedlium-wer",
    "score": 0.0386,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'Tedlium', column 'distil-v3' = 3.86 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "earnings22-wer",
    "score": 0.1179,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'Earnings22', column 'distil-v3' = 11.79 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "spgispeech-wer",
    "score": 0.0327,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Short-Form Evaluation' table (columns: Dataset | Size / h | large-v3 | large-v3-turbo | distil-v3 | distil-v3.5), row 'SPGISpeech', column 'distil-v3' = 3.27 (%). Setup printed for the table: 'We've evaluated the model on 5 in-distribution (ID) test sets and 2 out-of-distribution (OOD) test sets for short-form transcription, as done in Open ASR Leaderboard' and 'WER results shown here are post-normalization' (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is the Distil-Whisper team's own previous version (column header links https://huggingface.co/distil-whisper/distil-large-v3 in the summary table) -> catalogue distil-large-v3; by lab (same Hugging Face Distil-Whisper family, not marked as copied). The four columns in this table are numerically identical to the HF Open ASR Leaderboard CSV (github.com/huggingface/open_asr_leaderboard @16743c6 en_shortform.csv), which Hugging Face (the same organisation; card authors Le Bihan, Zheng and Srivastav are leaderboard maintainers) runs, so this is HF's own pipeline measurement. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v2",
    "bench": "tedlium-longform-wer",
    "score": 0.0966,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'tedlium-long-form', column 'distil-v2' = 9.66 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (Distil-Large-v2 linked in the card Training Details (huggingface.co/distil-whisper/distil-large-v2)) -> catalogue distil-large-v2; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v2",
    "bench": "meanwhile-longform-wer",
    "score": 0.1675,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'meanwhile', column 'distil-v2' = 16.75 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (Distil-Large-v2 linked in the card Training Details (huggingface.co/distil-whisper/distil-large-v2)) -> catalogue distil-large-v2; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v2",
    "bench": "earnings21-longform-wer",
    "score": 0.1509,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'earnings21', column 'distil-v2' = 15.09 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (Distil-Large-v2 linked in the card Training Details (huggingface.co/distil-whisper/distil-large-v2)) -> catalogue distil-large-v2; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v2",
    "bench": "earnings22-longform-wer",
    "score": 0.1911,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'earnings22', column 'distil-v2' = 19.11 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (Distil-Large-v2 linked in the card Training Details (huggingface.co/distil-whisper/distil-large-v2)) -> catalogue distil-large-v2; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v2",
    "bench": "rev16-longform-wer",
    "score": 0.2115,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'rev16', column 'distil-v2' = 21.15 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (Distil-Large-v2 linked in the card Training Details (huggingface.co/distil-whisper/distil-large-v2)) -> catalogue distil-large-v2; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "tedlium-longform-wer",
    "score": 0.039,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'tedlium-long-form', column 'distil-v3' = 3.9 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (distil-large-v3 linked in the summary table (huggingface.co/distil-whisper/distil-large-v3)) -> catalogue distil-large-v3; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "meanwhile-longform-wer",
    "score": 0.0704,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'meanwhile', column 'distil-v3' = 7.04 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (distil-large-v3 linked in the summary table (huggingface.co/distil-whisper/distil-large-v3)) -> catalogue distil-large-v3; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "earnings21-longform-wer",
    "score": 0.1054,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'earnings21', column 'distil-v3' = 10.54 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (distil-large-v3 linked in the summary table (huggingface.co/distil-whisper/distil-large-v3)) -> catalogue distil-large-v3; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "earnings22-longform-wer",
    "score": 0.1506,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'earnings22', column 'distil-v3' = 15.06 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (distil-large-v3 linked in the summary table (huggingface.co/distil-whisper/distil-large-v3)) -> catalogue distil-large-v3; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "distil-large-v3",
    "bench": "rev16-longform-wer",
    "score": 0.1376,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "lab",
    "date": "2026-04-13",
    "note": "distil-large-v3.5 card @728a769, 'Performance' > 'Long-Form Evaluation' table (columns: Dataset | Size / h | large-v3-turbo | distil-v2 | distil-v3 | distil-v3.5), row 'rev16', column 'distil-v3' = 13.76 (%). Setup printed for the table: 'using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; WER post-normalisation (Whisper basic normaliser). Same table and setting as the existing distil-v3.5 row. Comparator is an earlier Distil-Whisper English model by the same HF team (distil-large-v3 linked in the summary table (huggingface.co/distil-whisper/distil-large-v3)) -> catalogue distil-large-v3; by lab. These long-form cells are not copies of the Open ASR long-form CSV (e.g. distil-v3 earnings22 15.06 here vs 14.98 in en_longform.csv @16743c6), consistent with the card's own sequential-decoding run. Converted from percent.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "cedit-bench-en-overall",
    "score": 0.653,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'FLUX.1 Kontext [Pro]' = 6.53. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "flux-1-kontext-pro",
    "bench": "cedit-bench-cn-overall",
    "score": 0.143,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'FLUX.1 Kontext [Pro]' = 1.43. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "gpt-image-1",
    "bench": "cedit-bench-en-overall",
    "score": 0.817,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'GPT Image 1 [High]' = 8.17. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10. Quality setting 'High' as printed.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "gpt-image-1",
    "bench": "cedit-bench-cn-overall",
    "score": 0.821,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'GPT Image 1 [High]' = 8.21. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10. Quality setting 'High' as printed.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "nano-banana",
    "bench": "cedit-bench-en-overall",
    "score": 0.72,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'Nano Banana' = 7.2. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10. The text names Nano Banana as Gemini-2.5-flash-image.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "nano-banana",
    "bench": "cedit-bench-cn-overall",
    "score": 0.736,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'Nano Banana' = 7.36. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10. The text names Nano Banana as Gemini-2.5-flash-image.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "seedream-4-0",
    "bench": "cedit-bench-en-overall",
    "score": 0.758,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'Seedream 4.0' = 7.58. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "seedream-4-0",
    "bench": "cedit-bench-cn-overall",
    "score": 0.757,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'Seedream 4.0' = 7.57. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "cedit-bench-en-overall",
    "score": 0.593,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'FLUX.1 Kontext [Dev]' = 5.93. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "cedit-bench-cn-overall",
    "score": 0.151,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'FLUX.1 Kontext [Dev]' = 1.51. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "qwen-image-edit",
    "bench": "cedit-bench-en-overall",
    "score": 0.752,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'Qwen-Image-Edit' = 7.52. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "qwen-image-edit",
    "bench": "cedit-bench-cn-overall",
    "score": 0.746,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'Qwen-Image-Edit' = 7.46. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "cedit-bench-en-overall",
    "score": 0.748,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'Qwen-Image-Edit [2509]' = 7.48. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "cedit-bench-cn-overall",
    "score": 0.737,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'Qwen-Image-Edit [2509]' = 7.37. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "longcat-image-edit",
    "bench": "cedit-bench-en-overall",
    "score": 0.767,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-EN G_O, row 'LongCat-Image-Edit' = 7.67. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "longcat-image-edit",
    "bench": "cedit-bench-cn-overall",
    "score": 0.765,
    "source": "https://arxiv.org/html/2512.07584v1",
    "by": "lab",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan), arXiv:2512.07584v1, CC BY 4.0, Table 9 'Performance comparison on CEdit-Bench', column CEdit-Bench-CN G_O, row 'LongCat-Image-Edit' = 7.65. CEdit-Bench is introduced in this report (§6.5.1) and judged with GPT-4o, so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "gemini-2-0-flash-image",
    "bench": "omnicontext-average",
    "score": 0.362,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "independent",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'Gemini-2.0-Flash-Image' = 3.62. OmniContext is introduced in this report (§5.4; GPT-4.1 judge), so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "nano-banana",
    "bench": "omnicontext-average",
    "score": 0.784,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "independent",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'Gemini-2.5-Flash-Image' = 7.84. OmniContext is introduced in this report (§5.4; GPT-4.1 judge), so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "bagel",
    "bench": "omnicontext-average",
    "score": 0.573,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "independent",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'BAGEL' = 5.73. OmniContext is introduced in this report (§5.4; GPT-4.1 judge), so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "qwen-image-edit-2509",
    "bench": "omnicontext-average",
    "score": 0.769,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "independent",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'Qwen-Image-Edit-2509' = 7.69. OmniContext is introduced in this report (§5.4; GPT-4.1 judge), so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "omnigen2",
    "bench": "omnicontext-average",
    "score": 0.795,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "lab",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'OmniGen2' = 7.95. OmniContext is introduced in this report (§5.4; GPT-4.1 judge), so every cell is the authors' own run. 0–10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-zh-eres2net-sim",
    "score": 0.756,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-zh SIM, row 'VoxCPM2', value in parentheses (ERes2Net) = 0.756. Marked '†': 'ERes2Net SIM scores were computed by us using the publicly released models'.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-eres2net-sim",
    "score": 0.725,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-en SIM, row 'VoxCPM2', value in parentheses (ERes2Net) = 0.725. Marked '†': 'ERes2Net SIM scores were computed by us using the publicly released models'.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-zh-eres2net-sim",
    "score": 0.818,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-zh SIM, row 'Dots.TTS-2B (SOAR)', value in parentheses (ERes2Net) = 0.818. Marked '†': 'ERes2Net SIM scores were computed by us using the publicly released models'.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "seed-tts-en-eres2net-sim",
    "score": 0.792,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-en SIM, row 'Dots.TTS-2B (SOAR)', value in parentheses (ERes2Net) = 0.792. Marked '†': 'ERes2Net SIM scores were computed by us using the publicly released models'.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "seed-tts-zh-eres2net-sim",
    "score": 0.847,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-zh SIM, row 'Qwen-Audio-3.0-TTS', value in parentheses (ERes2Net) = 0.847. The authors' own model.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "seed-tts-en-eres2net-sim",
    "score": 0.815,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS report (Alibaba), arXiv:2607.23938v1, CC BY 4.0, Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-en SIM, row 'Qwen-Audio-3.0-TTS', value in parentheses (ERes2Net) = 0.815. The authors' own model.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "hidream-o1-image",
    "bench": "longtext-en",
    "score": 0.979,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image README @2c2d29f (MIT), 'LongText-Bench — long-text rendering, EN & ZH' table, row 'HiDream-O1-Image', column 'LongText-Bench-EN' = 0.979. HiDream's own model; other makers' rows not used.",
    "source_id": "visual-hidream"
  },
  {
    "model": "hidream-o1-image",
    "bench": "longtext-cn",
    "score": 0.978,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image README @2c2d29f (MIT), 'LongText-Bench — long-text rendering, EN & ZH' table, row 'HiDream-O1-Image', column 'LongText-Bench-ZH' = 0.978. HiDream's own model; other makers' rows not used.",
    "source_id": "visual-hidream"
  },
  {
    "model": "hidream-i1-full",
    "bench": "longtext-en",
    "score": 0.543,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image README @2c2d29f (MIT), 'LongText-Bench — long-text rendering, EN & ZH' table, row 'HiDream-I1-Full', column 'LongText-Bench-EN' = 0.543. HiDream's own model; other makers' rows not used.",
    "source_id": "visual-hidream"
  },
  {
    "model": "hidream-i1-full",
    "bench": "longtext-cn",
    "score": 0.024,
    "source": "https://github.com/HiDream-ai/HiDream-O1-Image/blob/2c2d29ff729e48f33e41f49edfdbd81d5ac103b4/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "HiDream-O1-Image README @2c2d29f (MIT), 'LongText-Bench — long-text rendering, EN & ZH' table, row 'HiDream-I1-Full', column 'LongText-Bench-ZH' = 0.024. HiDream's own model; other makers' rows not used.",
    "source_id": "visual-hidream"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.938,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'MMLU-Redux', section 'Knowledge' = 93.8; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "gpqa-unspecified",
    "score": 0.811,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'GPQA', section 'Knowledge' = 81.1; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'SuperGPQA', section 'Knowledge' = 64.9; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'HMMT25', section 'Reasoning' = 83.9; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'LiveBench 20241125', section 'Reasoning' = 78.4; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ojbench",
    "score": 0.325,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'OJBench', section 'Coding' = 32.5; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'IFEval', section 'Alignment' = 87.8; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "arena-hard-v2",
    "score": 0.797,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'Arena-Hard v2$', section 'Alignment' = 79.7; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "writingbench",
    "score": 0.883,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'WritingBench', section 'Alignment' = 88.3; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'BFCL-v3', section 'Agent' = 71.9; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU1-Retail', section 'Agent' = 67.8; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.46,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU1-Airline', section 'Agent' = 46.0; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Retail', section 'Agent' = 71.9; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Airline', section 'Agent' = 58.0; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.456,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'TAU2-Telecom', section 'Agent' = 45.6; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'MultiIF', section 'Multilingualism' = 80.6; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'MMLU-ProX', section 'Multilingualism' = 81.0; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "include",
    "score": 0.81,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'INCLUDE', section 'Multilingualism' = 81.0; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "polymath",
    "score": 0.601,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen own evaluation in the pinned model card Performance table (same table and row as the existing qwen3-235b-a22b hybrid-thinking row of this source, which is otherwise alone in its run). Own-model column of the card (Thinking-only checkpoint). Footnotes: Arena-Hard v2 win rates judged by GPT-4.1 ('$ For reproducibility, we report the win rates evaluated by GPT-4.1'); 'For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' GPQA subset, HMMT25 edition and IFEval aggregation not stated. TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-235B-A22B-Thinking-2507', row 'PolyMATH', section 'Multilingualism' = 60.1; percent converted to fraction. Checkpoint: the card's own model Qwen/Qwen3-235B-A22B-Thinking-2507 at the pinned revision = catalogue qwen3-235b-a22b-thinking-2507.",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "gemma-4-31b",
    "bench": "mcp-atlas",
    "score": 0.542,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'MCP Atlas (Public)', column 'Gemma4-31B Thinking Mode' = 54.2; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): MCP-Atlas: 500 public tasks, Gemini 2.5 Pro judge, mean pass rate (>0.75 threshold), averaged over 4 runs. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "mcp-atlas",
    "score": 0.625,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'MCP Atlas (Public)', column 'Qwen3.6-27B Thinking Mode' = 62.5; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): MCP-Atlas: 500 public tasks, Gemini 2.5 Pro judge, mean pass rate (>0.75 threshold), averaged over 4 runs. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "deepsearchqa",
    "score": 0.617,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'DeepSearch QA', column 'Gemma4-31B Thinking Mode' = 61.7; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): DeepSearchQA: 900 questions, single browsing tool, gpt-oss-120b judge, F1; 'We used the same search engine we selected across all models. Results are averaged over 4 runs to reduce variance. We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "deepsearchqa",
    "score": 0.711,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'DeepSearch QA', column 'Qwen3.6-27B Thinking Mode' = 71.1; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): DeepSearchQA: 900 questions, single browsing tool, gpt-oss-120b judge, F1; 'We used the same search engine we selected across all models. Results are averaged over 4 runs to reduce variance. We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "wildclawbench",
    "score": 0.376,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'WildClawBench', column 'Gemma4-31B Thinking Mode' = 37.6; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): WildClawBench: 60 tasks, OpenClaw harness, unweighted mean of task scores over three runs. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "wildclawbench",
    "score": 0.432,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'WildClawBench', column 'Qwen3.6-27B Thinking Mode' = 43.2; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): WildClawBench: 60 tasks, OpenClaw harness, unweighted mean of task scores over three runs. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "gaia2",
    "score": 0.364,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'Gaia2', column 'Gemma4-31B Thinking Mode' = 36.4; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): Gaia2: public GitHub repo, 800 tasks, OpenClaw harness, gpt-oss-120b judge, mean over three runs. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "gaia2",
    "score": 0.4,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'Gaia2', column 'Qwen3.6-27B Thinking Mode' = 40.0; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): Gaia2: public GitHub repo, 800 tasks, OpenClaw harness, gpt-oss-120b judge, mean over three runs. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "skillsbench-with-skills",
    "score": 0.324,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'SkillsBench (with skills)', column 'Gemma4-31B Thinking Mode' = 32.4; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): SkillsBench: 86-task set, skill folders mounted, mean per-task reward across four attempts. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "skillsbench-with-skills",
    "score": 0.466,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'SkillsBench (with skills)', column 'Qwen3.6-27B Thinking Mode' = 46.6; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): SkillsBench: 86-task set, skill folders mounted, mean per-task reward across four attempts. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "osworld-verified",
    "score": 0.585,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'OSWorld-Verified', column 'Gemma4-31B Thinking Mode' = 58.5; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): OSWorld-Verified: 361-task split (excludes 8 Google Drive tasks), GUI-only, 200-step cap, mean over four attempts; Qwen3.6-27B and Muse use the Claude computer-use action space (computer_20251124), Gemma4-31B the Gemini 2.5 Flash computer-use interface. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "osworld-verified",
    "score": 0.756,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'OSWorld-Verified', column 'Qwen3.6-27B Thinking Mode' = 75.6; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): OSWorld-Verified: 361-task split (excludes 8 Google Drive tasks), GUI-only, 200-step cap, mean over four attempts; Qwen3.6-27B and Muse use the Claude computer-use action space (computer_20251124), Gemma4-31B the Gemini 2.5 Flash computer-use interface. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.369,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'SWE-Bench Pro', column 'Gemma4-31B Thinking Mode' = 36.9; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): SWE-Bench Pro: 731 tasks (original task set), bash + file-edit scaffold, average over 4 runs. 'For SWE-Bench Pro, we do not use the Qwen3.6-27B's self-reported score which was measured on a refined version of the benchmark as we evaluated on the original task set. We report internal run results for the remaining cases.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.502,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'SWE-Bench Pro', column 'Qwen3.6-27B Thinking Mode' = 50.2; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): SWE-Bench Pro: 731 tasks (original task set), bash + file-edit scaffold, average over 4 runs. 'For SWE-Bench Pro, we do not use the Qwen3.6-27B's self-reported score which was measured on a refined version of the benchmark as we evaluated on the original task set. We report internal run results for the remaining cases.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-verified",
    "score": 0.666,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'SWE-Bench Verified', column 'Gemma4-31B Thinking Mode' = 66.6; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): SWE-Bench Verified: 500 tasks, bash + file-edit scaffold, average over 4 runs. 'For SWE-Bench Verified, we report Qwen3.6-27B's self-reported score ... We report internal run results for the remaining cases.' (Qwen3.6-27B cell is copied, not used; Gemma4-31B cell is an internal run.) Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "screenspot-pro",
    "score": 0.759,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'ScreenSpot Pro', column 'Gemma4-31B Thinking Mode' = 75.9; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): ScreenSpot-Pro: full 1,581-example set with an iterative Python cropping tool, point-in-box pass@1 averaged over four samples. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "screenspot-pro",
    "score": 0.761,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'ScreenSpot Pro', column 'Qwen3.6-27B Thinking Mode' = 76.1; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): ScreenSpot-Pro: full 1,581-example set with an iterative Python cropping tool, point-in-box pass@1 averaged over four samples. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.725,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'OmniDocBench v1.5', column 'Gemma4-31B Thinking Mode' = 72.5; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): OmniDocBench v1.5: 1,355 prompts, internal implementation of the scoring protocol (two-component weighted average, Hungarian matching), averaged over 4 runs. 'We report internal run results for all 3 models.' Checkpoint: google/gemma-4-31B-it (catalogue gemma-4-31b) in thinking mode; methodology: 'For Gemma4-31B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=64 across all benchmarks'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.778,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'OmniDocBench v1.5', column 'Qwen3.6-27B Thinking Mode' = 77.8; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): OmniDocBench v1.5: 1,355 prompts, internal implementation of the scoring protocol (two-component weighted average, Hungarian matching), averaged over 4 runs. 'We report internal run results for all 3 models.' Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "ifbench-unspecified",
    "score": 0.708,
    "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Meta Superintelligence Lab's own run of a third-party model, Muse Glimmer model card 'Benchmarks' table, row 'IFBench', column 'Qwen3.6-27B Thinking Mode' = 70.8; percent converted to fraction. Provenance from the card's linked evaluation report https://research.meta.ai/static/muse-glimmer-methodology (PDF linked from the card: 'For more detail about our evaluations, see our report'; retrieved 2026-10-09, sha256 fb08920a...6c91): IFBench: 294 tasks, mean pass rate over 4 runs. 'We report internal run results for Qwen3.6-27B and our model, and self-reported number for Gemma4-31B.' (Gemma cell not used.) Checkpoint: Qwen/Qwen3.6-27B (catalogue qwen3-6-27b) in thinking mode; methodology: 'For Qwen3.6-27B, we follow the recommended sampling configuration and we use thinking mode and temperature=1.0/top_p=0.95/top_k=20 across all benchmarks, except for GAIA2 and WildClawBench, where we use temperature=0.6/top_p=0.95/top_k=20'.",
    "source_id": "speed-meta-muse-glimmer-30b"
  },
  {
    "model": "minimax-m2-7",
    "bench": "terminal-bench-2.1",
    "score": 0.555,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'MiniMax-2.7 230B-A10B' = 55.5; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "terminal-bench-2.1",
    "score": 0.593,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'GLM-5.1 744B-A40B' = 59.3; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "terminal-bench-2.1",
    "score": 0.672,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'Kimi-K2.6 1T-A32B' = 67.2; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "terminal-bench-2.1",
    "score": 0.499,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'Qwen-3.5 397B-17B' = 49.9; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "terminal-bench-2.1",
    "score": 0.492,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'DS-v4-Pro 1.6T-A49B' = 49.2; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "terminal-bench-2.1",
    "score": 0.542,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Terminal Bench 2.1', column 'DS-v4-Flash 284B-A13B' = 54.2; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "swe-bench-verified",
    "score": 0.753,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'MiniMax-2.7 230B-A10B' = 75.3; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-verified",
    "score": 0.762,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'GLM-5.1 744B-A40B' = 76.2; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "swe-bench-verified",
    "score": 0.757,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'Kimi-K2.6 1T-A32B' = 75.7; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-verified",
    "score": 0.736,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'Qwen-3.5 397B-17B' = 73.6; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "swe-bench-verified",
    "score": 0.745,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'DS-v4-Pro 1.6T-A49B' = 74.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "swe-bench-verified",
    "score": 0.735,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Verified', column 'DS-v4-Flash 284B-A13B' = 73.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "swe-bench-multilingual",
    "score": 0.718,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'MiniMax-2.7 230B-A10B' = 71.8; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-multilingual",
    "score": 0.748,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'GLM-5.1 744B-A40B' = 74.8; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "swe-bench-multilingual",
    "score": 0.771,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'Kimi-K2.6 1T-A32B' = 77.1; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-multilingual",
    "score": 0.709,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'Qwen-3.5 397B-17B' = 70.9; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "swe-bench-multilingual",
    "score": 0.765,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'DS-v4-Pro 1.6T-A49B' = 76.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "swe-bench-multilingual",
    "score": 0.75,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SWE-Bench Multilingual', column 'DS-v4-Flash 284B-A13B' = 75.0; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "tau3-bench",
    "score": 0.661,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'MiniMax-2.7 230B-A10B' = 66.1; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "tau3-bench",
    "score": 0.697,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'GLM-5.1 744B-A40B' = 69.7; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "tau3-bench",
    "score": 0.724,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'Kimi-K2.6 1T-A32B' = 72.4; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-bench",
    "score": 0.71,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'Qwen-3.5 397B-17B' = 71.0; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "tau3-bench",
    "score": 0.732,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'DS-v4-Pro 1.6T-A49B' = 73.2; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "tau3-bench",
    "score": 0.737,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Average', column 'DS-v4-Flash 284B-A13B' = 73.7; percent converted to fraction. TauBench V3 average over Airline, Retail, Telecom and Banking; Appendix A.1: additional user-simulation prompt from tau2-bench-revised, banking (terminal_use) setting, 'DeepSeek-V4 models were evaluated under max reasoning. User simulator: GPT-5.2 (low reasoning effort). 8 trial average.' Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "browsecomp",
    "score": 0.541,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'MiniMax-2.7 230B-A10B' = 54.1; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "browsecomp",
    "score": 0.594,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'GLM-5.1 744B-A40B' = 59.4; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "browsecomp",
    "score": 0.613,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'Kimi-K2.6 1T-A32B' = 61.3; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "browsecomp",
    "score": 0.405,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'Qwen-3.5 397B-17B' = 40.5; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "browsecomp",
    "score": 0.594,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'DS-v4-Pro 1.6T-A49B' = 59.4; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "browsecomp",
    "score": 0.469,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'BrowseComp', column 'DS-v4-Flash 284B-A13B' = 46.9; percent converted to fraction. Appendix A.1: custom agentic search harness with Tavily search/browse and terminal access, results persisted to a per-task disk workspace. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.772,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'MiniMax-2.7 230B-A10B' = 77.2; percent converted to fraction. v6, date window not stated. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.857,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'GLM-5.1 744B-A40B' = 85.7; percent converted to fraction. v6, date window not stated. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.902,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'Kimi-K2.6 1T-A32B' = 90.2; percent converted to fraction. v6, date window not stated. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.793,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'Qwen-3.5 397B-17B' = 79.3; percent converted to fraction. v6, date window not stated. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.925,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'DS-v4-Pro 1.6T-A49B' = 92.5; percent converted to fraction. v6, date window not stated. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.909,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'LiveCodeBench (v6)', column 'DS-v4-Flash 284B-A13B' = 90.9; percent converted to fraction. v6, date window not stated. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "imo-answerbench",
    "score": 0.683,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'MiniMax-2.7 230B-A10B' = 68.3; percent converted to fraction. No tools. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "imo-answerbench",
    "score": 0.868,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'GLM-5.1 744B-A40B' = 86.8; percent converted to fraction. No tools. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "imo-answerbench",
    "score": 0.911,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'Kimi-K2.6 1T-A32B' = 91.1; percent converted to fraction. No tools. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "imo-answerbench",
    "score": 0.831,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'Qwen-3.5 397B-17B' = 83.1; percent converted to fraction. No tools. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "imo-answerbench",
    "score": 0.93,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'DS-v4-Pro 1.6T-A49B' = 93.0; percent converted to fraction. No tools. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "imo-answerbench",
    "score": 0.911,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IMOAnswerBench (no tools)', column 'DS-v4-Flash 284B-A13B' = 91.1; percent converted to fraction. No tools. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "gpqa-unspecified",
    "score": 0.866,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'MiniMax-2.7 230B-A10B' = 86.6; percent converted to fraction. No tools; subset not stated. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "gpqa-unspecified",
    "score": 0.861,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'GLM-5.1 744B-A40B' = 86.1; percent converted to fraction. No tools; subset not stated. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "gpqa-unspecified",
    "score": 0.91,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'Kimi-K2.6 1T-A32B' = 91.0; percent converted to fraction. No tools; subset not stated. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "gpqa-unspecified",
    "score": 0.871,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'Qwen-3.5 397B-17B' = 87.1; percent converted to fraction. No tools; subset not stated. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "gpqa-unspecified",
    "score": 0.878,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'DS-v4-Pro 1.6T-A49B' = 87.8; percent converted to fraction. No tools; subset not stated. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "gpqa-unspecified",
    "score": 0.885,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'GPQA (no tools)', column 'DS-v4-Flash 284B-A13B' = 88.5; percent converted to fraction. No tools; subset not stated. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "scicode-subtask",
    "score": 0.383,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'MiniMax-2.7 230B-A10B' = 38.3; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "scicode-subtask",
    "score": 0.477,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'GLM-5.1 744B-A40B' = 47.7; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "scicode-subtask",
    "score": 0.52,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'Kimi-K2.6 1T-A32B' = 52.0; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "scicode-subtask",
    "score": 0.48,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'Qwen-3.5 397B-17B' = 48.0; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "scicode-subtask",
    "score": 0.505,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'DS-v4-Pro 1.6T-A49B' = 50.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "scicode-subtask",
    "score": 0.482,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'SciCode (subtask)', column 'DS-v4-Flash 284B-A13B' = 48.2; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "hle",
    "score": 0.231,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'MiniMax-2.7 230B-A10B' = 23.1; percent converted to fraction. No tools. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "hle",
    "score": 0.272,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'GLM-5.1 744B-A40B' = 27.2; percent converted to fraction. No tools. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "hle",
    "score": 0.348,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'Kimi-K2.6 1T-A32B' = 34.8; percent converted to fraction. No tools. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle",
    "score": 0.285,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'Qwen-3.5 397B-17B' = 28.5; percent converted to fraction. No tools. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "hle",
    "score": 0.377,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'DS-v4-Pro 1.6T-A49B' = 37.7; percent converted to fraction. No tools. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "hle",
    "score": 0.322,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (no tools)', column 'DS-v4-Flash 284B-A13B' = 32.2; percent converted to fraction. No tools. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "hle-with-tools",
    "score": 0.504,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (with tools)', column 'GLM-5.1 744B-A40B' = 50.4; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "hle-with-tools",
    "score": 0.54,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (with tools)', column 'Kimi-K2.6 1T-A32B' = 54.0; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle-with-tools",
    "score": 0.483,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (with tools)', column 'Qwen-3.5 397B-17B' = 48.3; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "hle-with-tools",
    "score": 0.482,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (with tools)', column 'DS-v4-Pro 1.6T-A49B' = 48.2; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "hle-with-tools",
    "score": 0.451,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'HLE (with tools)', column 'DS-v4-Flash 284B-A13B' = 45.1; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "critpt",
    "score": 0.031,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'N-3-Ultra 550B-A55B' = 3.1; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: own model, column 'N-3-Ultra 550B-A55B' (BF16 checkpoint, as for the existing rows of this source).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "critpt",
    "score": 0.006,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'MiniMax-2.7 230B-A10B' = 0.6; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "critpt",
    "score": 0.037,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'GLM-5.1 744B-A40B' = 3.7; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "critpt",
    "score": 0.091,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'Kimi-K2.6 1T-A32B' = 9.1; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "critpt",
    "score": 0.024,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'Qwen-3.5 397B-17B' = 2.4; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "critpt",
    "score": 0.14,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'DS-v4-Pro 1.6T-A49B' = 14.0; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "critpt",
    "score": 0.106,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'CritPt (no tools)', column 'DS-v4-Flash 284B-A13B' = 10.6; percent converted to fraction. Extra row (benchmark id exists; own-model cell added too). No tools. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "mmlu-pro",
    "score": 0.819,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'MiniMax-2.7 230B-A10B' = 81.9; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "mmlu-pro",
    "score": 0.859,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'GLM-5.1 744B-A40B' = 85.9; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "mmlu-pro",
    "score": 0.881,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'Kimi-K2.6 1T-A32B' = 88.1; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-pro",
    "score": 0.883,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'Qwen-3.5 397B-17B' = 88.3; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "mmlu-pro",
    "score": 0.875,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'DS-v4-Pro 1.6T-A49B' = 87.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "mmlu-pro",
    "score": 0.864,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-Pro', column 'DS-v4-Flash 284B-A13B' = 86.4; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "ifbench-prompt-loose",
    "score": 0.746,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'MiniMax-2.7 230B-A10B' = 74.6; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "ifbench-prompt-loose",
    "score": 0.766,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'GLM-5.1 744B-A40B' = 76.6; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "ifbench-prompt-loose",
    "score": 0.737,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'Kimi-K2.6 1T-A32B' = 73.7; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ifbench-prompt-loose",
    "score": 0.782,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'Qwen-3.5 397B-17B' = 78.2; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "ifbench-prompt-loose",
    "score": 0.791,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'DS-v4-Pro 1.6T-A49B' = 79.1; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "ifbench-prompt-loose",
    "score": 0.82,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'IFBench (prompt loose)', column 'DS-v4-Flash 284B-A13B' = 82.0; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "multichallenge",
    "score": 0.425,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'MiniMax-2.7 230B-A10B' = 42.5; percent converted to fraction. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "multichallenge",
    "score": 0.63,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'GLM-5.1 744B-A40B' = 63.0; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "multichallenge",
    "score": 0.631,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'Kimi-K2.6 1T-A32B' = 63.1; percent converted to fraction. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "multichallenge",
    "score": 0.639,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'Qwen-3.5 397B-17B' = 63.9; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "multichallenge",
    "score": 0.641,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'DS-v4-Pro 1.6T-A49B' = 64.1; percent converted to fraction. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "multichallenge",
    "score": 0.635,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Multi-Challenge', column 'DS-v4-Flash 284B-A13B' = 63.5; percent converted to fraction. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "longbench-v2",
    "score": 0.689,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Longbench v2 (\\leq 1M)', column 'Qwen-3.5 397B-17B' = 68.9; percent converted to fraction. LongBench v2 items up to 1M tokens. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "longbench-v2",
    "score": 0.621,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Longbench v2 (\\leq 1M)', column 'DS-v4-Pro 1.6T-A49B' = 62.1; percent converted to fraction. LongBench v2 items up to 1M tokens. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "longbench-v2",
    "score": 0.57,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'Longbench v2 (\\leq 1M)', column 'DS-v4-Flash 284B-A13B' = 57.0; percent converted to fraction. LongBench v2 items up to 1M tokens. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "mmlu-prox",
    "score": 0.784,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'MiniMax-2.7 230B-A10B' = 78.4; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: MiniMaxAI/MiniMax-M2.7 (catalogue minimax-m2-7; 230B-A10B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "mmlu-prox",
    "score": 0.858,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'GLM-5.1 744B-A40B' = 85.8; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: zai-org/GLM-5.1 (catalogue glm-5-1; 744B-A40B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "mmlu-prox",
    "score": 0.85,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'Kimi-K2.6 1T-A32B' = 85.0; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: moonshotai/Kimi-K2.6 (catalogue kimi-k2-6; 1T-A32B as printed).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-prox",
    "score": 0.864,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'Qwen-3.5 397B-17B' = 86.4; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: Qwen/Qwen3.5-397B-A17B (catalogue qwen3-5-397b-a17b; printed 397B-17B).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "mmlu-prox",
    "score": 0.856,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'DS-v4-Pro 1.6T-A49B' = 85.6; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: DeepSeek-V4-Pro 1.6T-A49B; report v1 dated 2026-06-12, before the 0813 GA, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Pro (catalogue deepseek-v4-pro-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "mmlu-prox",
    "score": 0.843,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA's own runs, Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0) Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.' Section 3.7: 'All models were evaluated under the same evaluation settings including agentic resources (CPU compute and timeouts), input data, prompt templates, number of repeats, and extraction/metric implementation. Inference parameters such as temperature and top_p, and maximum number of tokens were directly extracted from the recommendations of the corresponding model cards.' Row 'MMLU-ProX (avg en/de/fr/es/it/ja/zh/hi/pt/ko)', column 'DS-v4-Flash 284B-A13B' = 84.3; percent converted to fraction. Average over en/de/fr/es/it/ja/zh/hi/pt/ko. Checkpoint: DeepSeek-V4-Flash 284B-A13B; report v1 dated 2026-06-12, before the 0731 release, so the April 2026 preview weights deepseek-ai/DeepSeek-V4-Flash (catalogue deepseek-v4-flash-preview).",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-4-7",
    "bench": "gpqa-diamond",
    "score": 0.857,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'GPQA-Diamond' = 85.7; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "hle",
    "score": 0.248,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'HLE' = 24.8; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "hle-with-tools",
    "score": 0.428,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'HLE (w/ Tools)' = 42.8; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "hmmt-nov-2025",
    "score": 0.935,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'HMMT Nov. 2025' = 93.5; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "imo-answerbench",
    "score": 0.82,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'IMOAnswerBench' = 82.0; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "swe-bench-verified",
    "score": 0.738,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'SWE-bench Verified' = 73.8; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "swe-bench-multilingual",
    "score": 0.667,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'SWE-bench Multilingual' = 66.7; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "terminal-bench-2",
    "score": 0.41,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'Terminal Bench 2.0' = 41.0; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp",
    "score": 0.52,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'BrowseComp' = 52.0; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp-context-managed",
    "score": 0.675,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'BrowseComp (w/ Context Manage)' = 67.5; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "browsecomp-zh",
    "score": 0.666,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'BrowseComp-Zh' = 66.6; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-7",
    "bench": "tau2-bench",
    "score": 0.874,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2026-01-29",
    "note": "Z.ai own evaluation in the pinned GLM-4.7 model card benchmark table (own-model column; the GLM-4.6 column of the same table was already recorded and was alone in its run). GLM evaluation parameters (card 'Evaluation Parameters'): default temperature=1.0, top_p=0.95, max_new_tokens=131072; Terminal Bench and SWE-bench Verified temperature=0.7, top_p=1.0, max_new_tokens=16384; τ²-Bench temperature 0 with an extra user prompt for Retail/Telecom and the Claude Opus 4.5 airline fixes; Preserved Thinking for τ²-Bench and Terminal Bench 2. Column 'GLM-4.7', row 'τ²-Bench' = 87.4; percent converted to fraction. Checkpoint: zai-org/GLM-4.7 at the pinned revision = catalogue glm-4-7 (thinking on by default).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "imo-answerbench",
    "score": 0.753,
    "source": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Meituan LongCat team's in-house run of a third-party model, LongCat-2.0 model card 'Evaluation Results' table: 'Unless noted with `*`, all scores are measured in-house under a unified harness.' Notes: '`*` — cited from the model's official report; `-` — no comparable public score.' Only unstarred cells used. Reasoning effort / thinking budget of the comparator not printed (highest setting cannot be confirmed). Row 'IMO-AnswerBench', column 'Claude Opus 4.8' = 75.3; percent converted to fraction. Checkpoint: Claude Opus 4.8 (catalogue claude-opus-4-8; thinking setting not printed). Same printed value as the starred (cited) Claude Opus 4.6 cell of this row (75.3*); the Opus 4.8 cell itself is unstarred, i.e. in-house. Previous passes imported the other 23 unstarred comparator cells of this table but not this one.",
    "source_id": "card-meituan-longcat--longcat-2.0"
  },
  {
    "model": "glm-5",
    "bench": "hle",
    "score": 0.305,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5', row 'HLE' = 30.5; percent converted to fraction. Checkpoint: zai-org/GLM-5 (catalogue glm-5), same-lab column of the GLM-5.1 card.",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5",
    "bench": "gpqa-diamond",
    "score": 0.86,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5', row 'GPQA-Diamond' = 86.0; percent converted to fraction. Checkpoint: zai-org/GLM-5 (catalogue glm-5), same-lab column of the GLM-5.1 card.",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "hmmt-feb-2026",
    "score": 0.826,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5.1', row 'HMMT Feb. 2026' = 82.6; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 at the pinned revision = catalogue glm-5-1 (own model).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "nl2repo",
    "score": 0.427,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5.1', row 'NL2Repo' = 42.7; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 at the pinned revision = catalogue glm-5-1 (own model).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "terminal-bench-2",
    "score": 0.635,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5.1', row 'Terminal-Bench 2.0 (Terminus-2)' = 63.5; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 at the pinned revision = catalogue glm-5-1 (own model).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "mcp-atlas",
    "score": 0.718,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5.1', row 'MCP-Atlas (Public Set)' = 71.8; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 at the pinned revision = catalogue glm-5-1 (own model).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "glm-5-1",
    "bench": "toolathlon",
    "score": 0.407,
    "source": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Z.ai own evaluation in the pinned GLM-5.1 model card 'Benchmark' table (same-lab GLM-5.1 and GLM-5 columns; the other column of this row was already recorded and alone in its run). Settings not printed in the card beyond the row labels. Column 'GLM-5.1', row 'Tool-Decathlon' = 40.7; percent converted to fraction. Checkpoint: zai-org/GLM-5.1 at the pinned revision = catalogue glm-5-1 (own model).",
    "source_id": "card-zai-org--glm-5.1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "mmlu-pro",
    "score": 0.632,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLUPro', column 'Qwen 2.5 14b instruct' = 63.2; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu-pro",
    "score": 0.634,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLUPro', column 'GPT 4o-mini' = 63.4; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "mmlu-pro",
    "score": 0.644,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLUPro', column 'Llama-3.3 70b instruct' = 64.4; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "mmlu-pro",
    "score": 0.696,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLUPro', column 'Qwen 2.5 72b instruct' = 69.6; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "humaneval-plus",
    "score": 0.791,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval+', column 'Qwen 2.5 14b instruct' = 79.1; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "humaneval-plus",
    "score": 0.82,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval+', column 'GPT 4o-mini' = 82.0; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "humaneval-plus",
    "score": 0.779,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval+', column 'Llama-3.3 70b instruct' = 77.9; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "humaneval-plus",
    "score": 0.784,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval+', column 'Qwen 2.5 72b instruct' = 78.4; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "arena-hard-unspecified",
    "score": 0.702,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'ArenaHard', column 'Qwen 2.5 14b instruct' = 70.2; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "arena-hard-unspecified",
    "score": 0.762,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'ArenaHard', column 'GPT 4o-mini' = 76.2; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "arena-hard-unspecified",
    "score": 0.655,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'ArenaHard', column 'Llama-3.3 70b instruct' = 65.5; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "arena-hard-unspecified",
    "score": 0.784,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'ArenaHard', column 'Qwen 2.5 72b instruct' = 78.4; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "ifeval-unspecified",
    "score": 0.787,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'IFEval', column 'Qwen 2.5 14b instruct' = 78.7; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "ifeval-unspecified",
    "score": 0.8,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'IFEval', column 'GPT 4o-mini' = 80.0; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "ifeval-unspecified",
    "score": 0.893,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'IFEval', column 'Llama-3.3 70b instruct' = 89.3; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "ifeval-unspecified",
    "score": 0.85,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'IFEval', column 'Qwen 2.5 72b instruct' = 85.0; percent converted to fraction. Section 6: 'We also consider MMLU-pro, HumanEval+, ArenaHard, and IFEval, for which we use an internal framework and prompting and extraction.' The comparison columns are models 'we benchmark against' (Section 6); footnote 1 shows Microsoft ran the comparison models itself rather than copying vendor figures. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simple-evals-drop-f1",
    "score": 0.855,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'DROP', column 'Qwen 2.5 14b instruct' = 85.5; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-drop-f1",
    "score": 0.793,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'DROP', column 'GPT 4o-mini' = 79.3; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "simple-evals-drop-f1",
    "score": 0.902,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'DROP', column 'Llama-3.3 70b instruct' = 90.2; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "simple-evals-drop-f1",
    "score": 0.767,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'DROP', column 'Qwen 2.5 72b instruct' = 76.7; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simple-evals-mmlu",
    "score": 0.799,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLU', column 'Qwen 2.5 14b instruct' = 79.9; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-mmlu",
    "score": 0.818,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLU', column 'GPT 4o-mini' = 81.8; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "simple-evals-mmlu",
    "score": 0.863,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLU', column 'Llama-3.3 70b instruct' = 86.3; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "simple-evals-mmlu",
    "score": 0.853,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLU', column 'Qwen 2.5 72b instruct' = 85.3; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "phi-4",
    "bench": "simple-evals-mmlu",
    "score": 0.848,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MMLU', column 'phi-4 14b' = 84.8; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: microsoft/phi-4 (own model, column 'phi-4 14b').",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.429,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'GPQA', column 'Qwen 2.5 14b instruct' = 42.9; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.409,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'GPQA', column 'GPT 4o-mini' = 40.9; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.491,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'GPQA', column 'Llama-3.3 70b instruct' = 49.1; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.49,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'GPQA', column 'Qwen 2.5 72b instruct' = 49.0; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "phi-4",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.561,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'GPQA', column 'phi-4 14b' = 56.1; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: microsoft/phi-4 (own model, column 'phi-4 14b').",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simple-evals-humaneval",
    "score": 0.721,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval', column 'Qwen 2.5 14b instruct' = 72.1; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-humaneval",
    "score": 0.862,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval', column 'GPT 4o-mini' = 86.2; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "simple-evals-humaneval",
    "score": 0.789,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval', column 'Llama-3.3 70b instruct' = 78.9; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b). Cell carries footnote 1 (Microsoft's simple-evals run scores Llama lower than Meta's own figure).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "simple-evals-humaneval",
    "score": 0.804,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval', column 'Qwen 2.5 72b instruct' = 80.4; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "phi-4",
    "bench": "simple-evals-humaneval",
    "score": 0.826,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'HumanEval', column 'phi-4 14b' = 82.6; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: microsoft/phi-4 (own model, column 'phi-4 14b').",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "simple-evals-mgsm",
    "score": 0.796,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MGSM', column 'Qwen 2.5 14b instruct' = 79.6; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-14B-Instruct (catalogue qwen2-5-14b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "simple-evals-mgsm",
    "score": 0.865,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MGSM', column 'GPT 4o-mini' = 86.5; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: GPT-4o-mini; the only gpt-4o-mini snapshot before this Dec 2024 report is gpt-4o-mini-2024-07-18 (catalogue gpt-4o-mini-2024-07-18).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "simple-evals-mgsm",
    "score": 0.891,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MGSM', column 'Llama-3.3 70b instruct' = 89.1; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: meta-llama/Llama-3.3-70B-Instruct (catalogue llama-3-3-70b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "simple-evals-mgsm",
    "score": 0.873,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "independent",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MGSM', column 'Qwen 2.5 72b instruct' = 87.3; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: Qwen/Qwen2.5-72B-Instruct (catalogue qwen2-5-72b).",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "phi-4",
    "bench": "simple-evals-mgsm",
    "score": 0.806,
    "source": "https://arxiv.org/html/2412.08905v1",
    "by": "lab",
    "date": "2024-12-12",
    "note": "Microsoft Research's own runs, phi-4 technical report (arXiv 2412.08905v1, CC BY 4.0) Table 1, row 'MGSM', column 'phi-4 14b' = 80.6; percent converted to fraction. Table 1 caption: 'The first set of benchmarks uses OpenAI's simple-evals framework [24], specifying the prompts/extraction/temperature=0.5. We compare to small models of similar inference cost, as well as to larger models.' Section 6: 'We first report the values from OpenAI's simple-evals benchmark, which is a framework (including prompts, temperature, and extraction) for evaluating MMLU, GPQA diamond, MATH, HumanEval, MGSM, and the SimpleQA F1-score.' Footnote 1 on the Llama-3.3 cells: 'These scores are lower than those reported by Meta, perhaps because simple-evals has a strict formatting requirement that Llama models have particular trouble following. We use the simple-evals framework because it is reproducible, but Meta reports 77 for MATH and 88 for HumanEval on Llama-3.3.' (i.e. Microsoft ran the comparators itself). Extra row: benchmark id exists; own-model phi-4 cell added too. Checkpoint: microsoft/phi-4 (own model, column 'phi-4 14b').",
    "source_id": "report-2412.08905v1"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-pro",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen own evaluation in the pinned Qwen3.5-9B model card, language table (same table as the existing qwen3-5-9b row of this source; this source already holds the Qwen3.5-4B column for the table's other rows). Card: Qwen3.5 models operate in thinking mode by default. Column 'Qwen3.5-4B', row 'MMLU-Pro', section 'Knowledge & STEM' = 79.1; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-4B (catalogue qwen3-5-4b). The Qwen3.5-4B card (card-qwen--qwen3.5-4b @851bf6e) prints a byte-identical table, so its own lone rows for these three benchmarks duplicate these cells.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gpqa-diamond",
    "score": 0.762,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen own evaluation in the pinned Qwen3.5-9B model card, language table (same table as the existing qwen3-5-9b row of this source; this source already holds the Qwen3.5-4B column for the table's other rows). Card: Qwen3.5 models operate in thinking mode by default. Column 'Qwen3.5-4B', row 'GPQA Diamond', section 'Knowledge & STEM' = 76.2; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-4B (catalogue qwen3-5-4b). The Qwen3.5-4B card (card-qwen--qwen3.5-4b @851bf6e) prints a byte-identical table, so its own lone rows for these three benchmarks duplicate these cells.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6",
    "score": 0.558,
    "source": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen own evaluation in the pinned Qwen3.5-9B model card, language table (same table as the existing qwen3-5-9b row of this source; this source already holds the Qwen3.5-4B column for the table's other rows). Card: Qwen3.5 models operate in thinking mode by default. Column 'Qwen3.5-4B', row 'LiveCodeBench v6', section 'Reasoning & Coding' = 55.8; percent converted to fraction. Checkpoint: Qwen/Qwen3.5-4B (catalogue qwen3-5-4b). The Qwen3.5-4B card (card-qwen--qwen3.5-4b @851bf6e) prints a byte-identical table, so its own lone rows for these three benchmarks duplicate these cells.",
    "source_id": "card-qwen--qwen3.5-9b"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2024",
    "score": 0.813,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card 'Model Quality' table (same table as the existing phi-4-reasoning row; Phi-4-reasoning-plus is the same team's RL-tuned sibling). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Row 'Phi-4-reasoning-plus', column 'AIME 24' = 81.3; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only catalogue entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2025",
    "score": 0.78,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card 'Model Quality' table (same table as the existing phi-4-reasoning row; Phi-4-reasoning-plus is the same team's RL-tuned sibling). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Row 'Phi-4-reasoning-plus', column 'AIME 25' = 78.0; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only catalogue entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "omni-math",
    "score": 0.819,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card 'Model Quality' table (same table as the existing phi-4-reasoning row; Phi-4-reasoning-plus is the same team's RL-tuned sibling). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Row 'Phi-4-reasoning-plus', column 'OmniMath' = 81.9; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only catalogue entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "gpqa-diamond",
    "score": 0.689,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card 'Model Quality' table (same table as the existing phi-4-reasoning row; Phi-4-reasoning-plus is the same team's RL-tuned sibling). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Row 'Phi-4-reasoning-plus', column 'GPQA-D' = 68.9; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only catalogue entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "livecodebench-20240801-20250201",
    "score": 0.531,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card 'Model Quality' table (same table as the existing phi-4-reasoning row; Phi-4-reasoning-plus is the same team's RL-tuned sibling). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Row 'Phi-4-reasoning-plus', column 'LiveCodeBench (8/1/24–2/1/25)' = 53.1; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only catalogue entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "flenqa-3k",
    "score": 0.979,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card second 'Model Quality' table (same table as the existing phi-4 / phi-4-reasoning rows; extra rows for the proposed comparison-only Phi-4-reasoning-plus entry). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Column 'Phi-4-reasoning-plus', row 'FlenQA [3K-token subset]' = 97.9; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "ifeval-strict-level-unspecified",
    "score": 0.849,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card second 'Model Quality' table (same table as the existing phi-4 / phi-4-reasoning rows; extra rows for the proposed comparison-only Phi-4-reasoning-plus entry). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Column 'Phi-4-reasoning-plus', row 'IFEval Strict' = 84.9; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "arena-hard-unspecified",
    "score": 0.79,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card second 'Model Quality' table (same table as the existing phi-4 / phi-4-reasoning rows; extra rows for the proposed comparison-only Phi-4-reasoning-plus entry). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Column 'Phi-4-reasoning-plus', row 'ArenaHard' = 79.0; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "humaneval-plus",
    "score": 0.923,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card second 'Model Quality' table (same table as the existing phi-4 / phi-4-reasoning rows; extra rows for the proposed comparison-only Phi-4-reasoning-plus entry). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Column 'Phi-4-reasoning-plus', row 'HumanEvalPlus' = 92.3; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "mmlu-pro",
    "score": 0.76,
    "source": "https://huggingface.co/microsoft/Phi-4-reasoning/blob/1de18ec97600877ce63dbf60c73b998da99f0195/README.md",
    "by": "lab",
    "date": "2025-04-09",
    "note": "Microsoft Research's own evaluation in the pinned Phi-4-reasoning model card second 'Model Quality' table (same table as the existing phi-4 / phi-4-reasoning rows; extra rows for the proposed comparison-only Phi-4-reasoning-plus entry). Card: 'We evaluated Phi-4-reasoning using the open-source Eureka evaluation suite and our own internal benchmarks'. Column 'Phi-4-reasoning-plus', row 'MMLUPro' = 76.0; percent converted to fraction. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry phi-4-reasoning-plus).",
    "source_id": "card-microsoft--phi-4-reasoning"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau-bench-retail",
    "score": 0.603,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "independent",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'GPT-4o-0327', row 'TAU1-Retail', section 'Agent' = 60.3#; percent converted to fraction. Checkpoint: the footnote names the model actually run: GPT-4o-20241120 = catalogue gpt-4o-2024-11-20 (not the GPT-4o-0327 of the column header). Marker '#' in cell = footnote above.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau-bench-airline",
    "score": 0.428,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "independent",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'GPT-4o-0327', row 'TAU1-Airline', section 'Agent' = 42.8#; percent converted to fraction. Checkpoint: the footnote names the model actually run: GPT-4o-20241120 = catalogue gpt-4o-2024-11-20 (not the GPT-4o-0327 of the column header). Marker '#' in cell = footnote above.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-retail",
    "score": 0.667,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "independent",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'GPT-4o-0327', row 'TAU2-Retail', section 'Agent' = 66.7#; percent converted to fraction. Checkpoint: the footnote names the model actually run: GPT-4o-20241120 = catalogue gpt-4o-2024-11-20 (not the GPT-4o-0327 of the column header). Marker '#' in cell = footnote above.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-airline",
    "score": 0.42,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "independent",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'GPT-4o-0327', row 'TAU2-Airline', section 'Agent' = 42.0#; percent converted to fraction. Checkpoint: the footnote names the model actually run: GPT-4o-20241120 = catalogue gpt-4o-2024-11-20 (not the GPT-4o-0327 of the column header). Marker '#' in cell = footnote above.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-telecom",
    "score": 0.298,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "independent",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'GPT-4o-0327', row 'TAU2-Telecom', section 'Agent' = 29.8#; percent converted to fraction. Checkpoint: the footnote names the model actually run: GPT-4o-20241120 = catalogue gpt-4o-2024-11-20 (not the GPT-4o-0327 of the column header). Marker '#' in cell = footnote above.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau-bench-retail",
    "score": 0.591,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU1-Retail', section 'Agent' = 59.1; percent converted to fraction. Checkpoint: own model Qwen/Qwen3-30B-A3B-Instruct-2507 (non-thinking-only checkpoint); same values also printed in the Qwen3-Next-80B-A3B-Instruct card.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau-bench-airline",
    "score": 0.4,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU1-Airline', section 'Agent' = 40.0; percent converted to fraction. Checkpoint: own model Qwen/Qwen3-30B-A3B-Instruct-2507 (non-thinking-only checkpoint); same values also printed in the Qwen3-Next-80B-A3B-Instruct card.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.57,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Retail', section 'Agent' = 57.0; percent converted to fraction. Checkpoint: own model Qwen/Qwen3-30B-A3B-Instruct-2507 (non-thinking-only checkpoint); same values also printed in the Qwen3-Next-80B-A3B-Instruct card.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.38,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Airline', section 'Agent' = 38.0; percent converted to fraction. Checkpoint: own model Qwen/Qwen3-30B-A3B-Instruct-2507 (non-thinking-only checkpoint); same values also printed in the Qwen3-Next-80B-A3B-Instruct card.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.123,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation in the pinned Qwen3-30B-A3B-Instruct-2507 model card Performance table, Agent section (extra rows: benchmark ids exist; not one of the lone rows). Footnote on the GPT-4o-0327 TAU cells: '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' TAU1/TAU2 domains reported separately (no aggregate). Column 'Qwen3-30B-A3B-Instruct-2507', row 'TAU2-Telecom', section 'Agent' = 12.3; percent converted to fraction. Checkpoint: own model Qwen/Qwen3-30B-A3B-Instruct-2507 (non-thinking-only checkpoint); same values also printed in the Qwen3-Next-80B-A3B-Instruct card.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "olmo-2-7b",
    "bench": "math-subset-unspecified",
    "score": 0.301,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Instruct model card 'Evaluation' table (same table as the existing olmo-3-7b-instruct row; same-lab Olmo 2 column). Card: 'We evaluated our models on the following settings ... temperature 0.6, top_p 0.95, max_tokens 32768'; evaluation code OLMo-Eval. MATH subset not stated. Column '**Olmo 2 7B Instruct**', row 'MATH' = 30.1; percent converted to fraction. Checkpoint: Ai2's own previous model; 'Olmo 2 7B Instruct' = allenai/OLMo-2-1124-7B-Instruct, the only Olmo 2 7B Instruct release (catalogue olmo-2-7b).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "olmo-2-7b",
    "bench": "omega",
    "score": 0.052,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Ai2 own evaluation in the pinned Olmo-3-7B-Instruct model card 'Evaluation' table (same table as the existing olmo-3-7b-instruct row; same-lab Olmo 2 column). Card: 'We evaluated our models on the following settings ... temperature 0.6, top_p 0.95, max_tokens 32768'; evaluation code OLMo-Eval. MATH subset not stated. Column '**Olmo 2 7B Instruct**', row 'OMEGA' = 5.2; percent converted to fraction. Checkpoint: Ai2's own previous model; 'Olmo 2 7B Instruct' = allenai/OLMo-2-1124-7B-Instruct, the only Olmo 2 7B Instruct release (catalogue olmo-2-7b).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "math-subset-unspecified",
    "score": 0.961,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "lab",
    "date": "2026-01-05",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card 'Evaluation' table (same table as the existing olmo-3-1-32b-think row; same-lab columns). Card: 'We evaluated our models on the following settings'; evaluation code OLMo-Eval. MATH subset not stated. Column 'Olmo 3 Think 32B', row 'MATH', section 'Math' = 96.1; percent converted to fraction. Checkpoint: Ai2's own 'Olmo 3 Think 32B' = allenai/Olmo-3-32B-Think (catalogue olmo-3-32b-think), the final RLVR model, not the SFT/DPO stages.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "olmo-2-32b",
    "bench": "math-subset-unspecified",
    "score": 0.492,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "lab",
    "date": "2026-01-05",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card 'Evaluation' table (same table as the existing olmo-3-1-32b-think row; same-lab columns). Card: 'We evaluated our models on the following settings'; evaluation code OLMo-Eval. MATH subset not stated. Column 'Olmo 2 32B Instruct', row 'MATH', section 'Math' = 49.2; percent converted to fraction. Checkpoint: Ai2's own 'Olmo 2 32B Instruct' = allenai/OLMo-2-0325-32B-Instruct, the only Olmo 2 32B Instruct release (catalogue olmo-2-32b).",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "olmo-3-32b-think",
    "bench": "omega",
    "score": 0.508,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "lab",
    "date": "2026-01-05",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card 'Evaluation' table (same table as the existing olmo-3-1-32b-think row; same-lab columns). Card: 'We evaluated our models on the following settings'; evaluation code OLMo-Eval. MATH subset not stated. Column 'Olmo 3 Think 32B', row 'OMEGA', section 'Math' = 50.8; percent converted to fraction. Checkpoint: Ai2's own 'Olmo 3 Think 32B' = allenai/Olmo-3-32B-Think (catalogue olmo-3-32b-think), the final RLVR model, not the SFT/DPO stages.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "olmo-2-32b",
    "bench": "omega",
    "score": 0.098,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "lab",
    "date": "2026-01-05",
    "note": "Ai2 own evaluation in the pinned Olmo-3.1-32B-Think model card 'Evaluation' table (same table as the existing olmo-3-1-32b-think row; same-lab columns). Card: 'We evaluated our models on the following settings'; evaluation code OLMo-Eval. MATH subset not stated. Column 'Olmo 2 32B Instruct', row 'OMEGA', section 'Math' = 9.8; percent converted to fraction. Checkpoint: Ai2's own 'Olmo 2 32B Instruct' = allenai/OLMo-2-0325-32B-Instruct, the only Olmo 2 32B Instruct release (catalogue olmo-2-32b).",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "glm-5",
    "bench": "terminal-bench-2",
    "score": 0.562,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "lab",
    "date": "2026-08-11",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card benchmark table (own-model GLM-5 column; the same-lab GLM-4.7 cell of this row was already recorded and alone in its run). Footnotes: BrowseComp 'With context management, we use the same discard-all strategy as DeepSeek-v3.2 and Kimi K2.5'; Terminal-Bench 2.0 (Terminus 2) 'timeout=2h, temperature=0.7, top_p=1.0, max_new_tokens=8192, with a 128K context window'; for 'x / y †' cells the first value is the standard dataset ('†: A verified version of Terminal-Bench 2.0 that fixes some ambiguous instructions', not used). Column 'GLM-5', row 'Terminal-Bench 2.0 (Terminus 2)' = 56.2 / 60.7 †; percent converted to fraction. Checkpoint: zai-org/GLM-5 at the pinned revision = catalogue glm-5 (own model).",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "glm-5",
    "bench": "browsecomp-context-managed",
    "score": 0.759,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "lab",
    "date": "2026-08-11",
    "note": "Z.ai own evaluation in the pinned GLM-5 model card benchmark table (own-model GLM-5 column; the same-lab GLM-4.7 cell of this row was already recorded and alone in its run). Footnotes: BrowseComp 'With context management, we use the same discard-all strategy as DeepSeek-v3.2 and Kimi K2.5'; Terminal-Bench 2.0 (Terminus 2) 'timeout=2h, temperature=0.7, top_p=1.0, max_new_tokens=8192, with a 128K context window'; for 'x / y †' cells the first value is the standard dataset ('†: A verified version of Terminal-Bench 2.0 that fixes some ambiguous instructions', not used). Column 'GLM-5', row 'BrowseComp (w/ Context Manage)' = 75.9; percent converted to fraction. Checkpoint: zai-org/GLM-5 at the pinned revision = catalogue glm-5 (own model).",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "mcp-atlas",
    "score": 0.622,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Z.ai's own run of a third-party model, GLM-5 model card benchmark table, MCP-Atlas row (extra row: benchmark id exists, run already counts with GLM-5/GLM-4.7). Footnote: 'MCP-Atlas: All models are evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini 3 Pro as the judge model.' Column 'DeepSeek-V3.2', row 'MCP-Atlas (Public Set)' = 62.2; percent converted to fraction. Checkpoint: DeepSeek-V3.2 (catalogue deepseek-v3-2, the Dec 2025 release; not V3.2-Exp), think mode per footnote.",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "kimi-k2-5",
    "bench": "mcp-atlas",
    "score": 0.638,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Z.ai's own run of a third-party model, GLM-5 model card benchmark table, MCP-Atlas row (extra row: benchmark id exists, run already counts with GLM-5/GLM-4.7). Footnote: 'MCP-Atlas: All models are evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini 3 Pro as the judge model.' Column 'Kimi K2.5', row 'MCP-Atlas (Public Set)' = 63.8; percent converted to fraction. Checkpoint: Kimi K2.5 (catalogue kimi-k2-5), think mode per footnote.",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "mcp-atlas",
    "score": 0.652,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Z.ai's own run of a third-party model, GLM-5 model card benchmark table, MCP-Atlas row (extra row: benchmark id exists, run already counts with GLM-5/GLM-4.7). Footnote: 'MCP-Atlas: All models are evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini 3 Pro as the judge model.' Column 'Claude Opus 4.5', row 'MCP-Atlas (Public Set)' = 65.2; percent converted to fraction. Checkpoint: Claude Opus 4.5 (catalogue claude-opus-4-5), think mode per footnote; effort not printed.",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "mcp-atlas",
    "score": 0.666,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Z.ai's own run of a third-party model, GLM-5 model card benchmark table, MCP-Atlas row (extra row: benchmark id exists, run already counts with GLM-5/GLM-4.7). Footnote: 'MCP-Atlas: All models are evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini 3 Pro as the judge model.' Column 'Gemini 3 Pro', row 'MCP-Atlas (Public Set)' = 66.6; percent converted to fraction. Checkpoint: 'Gemini 3 Pro' before the GLM-5 release; the only Gemini 3 Pro API model then was gemini-3-pro-preview (catalogue gemini-3-pro-preview), think mode per footnote.",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "gpt-5-2",
    "bench": "mcp-atlas",
    "score": 0.68,
    "source": "https://huggingface.co/zai-org/GLM-5/blob/c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2/README.md",
    "by": "independent",
    "date": "2026-08-11",
    "note": "Z.ai's own run of a third-party model, GLM-5 model card benchmark table, MCP-Atlas row (extra row: benchmark id exists, run already counts with GLM-5/GLM-4.7). Footnote: 'MCP-Atlas: All models are evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini 3 Pro as the judge model.' Column 'GPT-5.2 (xhigh)', row 'MCP-Atlas (Public Set)' = 68.0; percent converted to fraction. Checkpoint: GPT-5.2 (catalogue gpt-5-2) at xhigh reasoning effort (column header 'GPT-5.2 (xhigh)').",
    "source_id": "card2-zai-org-glm-5"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hle",
    "score": 0.177,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/c0781d039fb7a1ba2abc4add0bdc293e92d2b8db/README.md",
    "by": "lab",
    "date": "2025-09-05",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-V3.1 model card, Evaluation table (same table and row as the existing deepseek-v3-1 HLE row; the R1 0528 column is a same-lab evaluation, already used from this card for 10 other rows; the same 17.7 is also printed in the DeepSeek-R1-0528 card). Note: 'HLE is evaluated with the text-only subset.' Column 'DeepSeek R1 0528', row 'Humanity's Last Exam (Pass@1)' = 17.7; percent converted to fraction. Checkpoint: deepseek-ai/DeepSeek-R1-0528 (catalogue deepseek-r1-0528).",
    "source_id": "card2-deepseek-ai-deepseek-v3-1"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.785,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-30B-A3B Thinking' = 78.5. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-pro",
    "score": 0.704,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-4B Thinking' = 70.4. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aime-2025",
    "score": 0.709,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'AIME25', column 'Qwen3-30B-A3B Thinking' = 70.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "aime-2025",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'AIME25', column 'Qwen3-4B Thinking' = 65.6. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-30B-A3B Thinking' = 57.4. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "livecodebench-v6",
    "score": 0.484,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-4B Thinking' = 48.4. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.363,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'Arena-Hard v2$', column 'Qwen3-30B-A3B Thinking' = 36.3. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "arena-hard-v2",
    "score": 0.137,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'Arena-Hard v2$', column 'Qwen3-4B Thinking' = 13.7. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'BFCL-v3', column 'Qwen3-30B-A3B Thinking' = 69.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "bfcl-v3",
    "score": 0.659,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'BFCL-v3', column 'Qwen3-4B Thinking' = 65.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "creative-writing-v3",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'Creative Writing v3', column 'Qwen3-30B-A3B Thinking' = 79.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "creative-writing-v3",
    "score": 0.611,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'Creative Writing v3', column 'Qwen3-4B Thinking' = 61.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "gpqa-unspecified",
    "score": 0.658,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'GPQA', column 'Qwen3-30B-A3B Thinking' = 65.8. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "gpqa-unspecified",
    "score": 0.559,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'GPQA', column 'Qwen3-4B Thinking' = 55.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.498,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'HMMT25', column 'Qwen3-30B-A3B Thinking' = 49.8. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.421,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'HMMT25', column 'Qwen3-4B Thinking' = 42.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.865,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'IFEval', column 'Qwen3-30B-A3B Thinking' = 86.5. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "ifeval-unspecified",
    "score": 0.819,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'IFEval', column 'Qwen3-4B Thinking' = 81.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "include",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'INCLUDE', column 'Qwen3-30B-A3B Thinking' = 71.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "include",
    "score": 0.618,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'INCLUDE', column 'Qwen3-4B Thinking' = 61.8. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livebench-20241125",
    "score": 0.743,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'LiveBench 20241125', column 'Qwen3-30B-A3B Thinking' = 74.3. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "livebench-20241125",
    "score": 0.636,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'LiveBench 20241125', column 'Qwen3-4B Thinking' = 63.6. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.731,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-ProX', column 'Qwen3-30B-A3B Thinking' = 73.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-prox",
    "score": 0.61,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-ProX', column 'Qwen3-4B Thinking' = 61.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-redux",
    "score": 0.895,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-Redux', column 'Qwen3-30B-A3B Thinking' = 89.5. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-redux",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MMLU-Redux', column 'Qwen3-4B Thinking' = 83.7. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multi-if-unspecified",
    "score": 0.722,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MultiIF', column 'Qwen3-30B-A3B Thinking' = 72.2. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "multi-if-unspecified",
    "score": 0.663,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'MultiIF', column 'Qwen3-4B Thinking' = 66.3. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ojbench",
    "score": 0.207,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'OJBench', column 'Qwen3-30B-A3B Thinking' = 20.7. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "ojbench",
    "score": 0.161,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'OJBench', column 'Qwen3-4B Thinking' = 16.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "polymath",
    "score": 0.461,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'PolyMATH', column 'Qwen3-30B-A3B Thinking' = 46.1. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "polymath",
    "score": 0.4,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'PolyMATH', column 'Qwen3-4B Thinking' = 40.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "supergpqa",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'SuperGPQA', column 'Qwen3-30B-A3B Thinking' = 51.8. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "supergpqa",
    "score": 0.427,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'SuperGPQA', column 'Qwen3-4B Thinking' = 42.7. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU1-Airline', column 'Qwen3-30B-A3B Thinking' = 32.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU1-Airline', column 'Qwen3-4B Thinking' = 32.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-retail",
    "score": 0.617,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU1-Retail', column 'Qwen3-30B-A3B Thinking' = 61.7. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau-bench-retail",
    "score": 0.339,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU1-Retail', column 'Qwen3-4B Thinking' = 33.9. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-airline",
    "score": 0.36,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Airline', column 'Qwen3-30B-A3B Thinking' = 36.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-airline",
    "score": 0.28,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Airline', column 'Qwen3-4B Thinking' = 28.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-retail",
    "score": 0.342,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Retail', column 'Qwen3-30B-A3B Thinking' = 34.2. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-retail",
    "score": 0.386,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Retail', column 'Qwen3-4B Thinking' = 38.6. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.228,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Telecom', column 'Qwen3-30B-A3B Thinking' = 22.8. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-telecom",
    "score": 0.175,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'TAU2-Telecom', column 'Qwen3-4B Thinking' = 17.5. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "writingbench",
    "score": 0.77,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'WritingBench', column 'Qwen3-30B-A3B Thinking' = 77.0. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-30B-A3B Thinking' = Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "writingbench",
    "score": 0.735,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/768f209d9ea81521153ed38c47d515654e938aea/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen3-4B-Thinking-2507 model card (Apache-2.0, pinned 768f209), '## Performance' table, row 'WritingBench', column 'Qwen3-4B Thinking' = 73.5. Same-lab comparison column in the card's own table (Qwen's own earlier model in Qwen's own table; no cell is marked as copied or credited elsewhere). Checkpoint: column 'Qwen3-4B Thinking' = Qwen/Qwen3-4B (April 2025 hybrid checkpoint) in thinking mode, matching the precedent for 'Qwen3-235B-A22B Thinking' columns in the Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Same table row and settings as the existing Qwen3-4B-Thinking-2507 row.",
    "source_id": "card-qwen--qwen3-4b-thinking-2507"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmlu-pro",
    "score": 0.762,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU-Pro', column 'OAI-OSS-20B*', cell '76.2' -> reproduced value 76.2. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.819,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU-Pro', column 'Qwen3-30B-A3B-Thinking-2507*', cell '81.9 (80.9)' -> reproduced value 81.9. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-pro",
    "score": 0.818,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU-Pro', column 'Qwen3-32B*', cell '81.8' -> reproduced value 81.8. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2025",
    "score": 0.903,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME25', column 'OAI-OSS-20B*', cell '90.3 (91.7)' -> reproduced value 90.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.813,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME25', column 'Qwen3-30B-A3B-Thinking-2507*', cell '81.3 (85)' -> reproduced value 81.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2025",
    "score": 0.733,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME25', column 'Qwen3-32B*', cell '73.3 (72.9)' -> reproduced value 73.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "hle",
    "score": 0.127,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'HLE', column 'OAI-OSS-20B*', cell '12.7 (10.9)' -> reproduced value 12.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "hle",
    "score": 0.087,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'HLE', column 'Qwen3-30B-A3B-Thinking-2507*', cell '8.7' -> reproduced value 8.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "hle",
    "score": 0.069,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'HLE', column 'Qwen3-32B*', cell '6.9' -> reproduced value 6.9. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "livecodebench-v6",
    "score": 0.638,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'LiveCodeBench v6 (02/2025-05/2025)', column 'OAI-OSS-20B*', cell '63.8' -> reproduced value 63.8. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.603,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'LiveCodeBench v6 (02/2025-05/2025)', column 'Qwen3-30B-A3B-Thinking-2507*', cell '60.3 (66)' -> reproduced value 60.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "livecodebench-v6",
    "score": 0.534,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'LiveCodeBench v6 (02/2025-05/2025)', column 'Qwen3-32B*', cell '53.4' -> reproduced value 53.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "swe-bench-verified",
    "score": 0.31,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SWE-Bench Verified (OpenHands)', column 'Qwen3-30B-A3B-Thinking-2507*', cell '31' -> reproduced value 31. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "swe-bench-verified",
    "score": 0.234,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SWE-Bench Verified (OpenHands)', column 'Qwen3-32B*', cell '23.4' -> reproduced value 23.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "aime-2024",
    "score": 0.927,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME24', column 'OAI-OSS-20B*', cell '92.7 (92.1)' -> reproduced value 92.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "aime-2024",
    "score": 0.877,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME24', column 'Qwen3-30B-A3B-Thinking-2507*', cell '87.7' -> reproduced value 87.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2024",
    "score": 0.827,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'AIME24', column 'Qwen3-32B*', cell '82.7 (81.4)' -> reproduced value 82.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "beyondaime",
    "score": 0.69,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'BeyondAIME', column 'OAI-OSS-20B*', cell '69' -> reproduced value 69. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "beyondaime",
    "score": 0.56,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'BeyondAIME', column 'Qwen3-30B-A3B-Thinking-2507*', cell '56' -> reproduced value 56. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "beyondaime",
    "score": 0.29,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'BeyondAIME', column 'Qwen3-32B*', cell '29' -> reproduced value 29. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "gpqa-diamond",
    "score": 0.722,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'GPQA-D', column 'OAI-OSS-20B*', cell '72.2 (71.5)' -> reproduced value 72.2. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.714,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'GPQA-D', column 'Qwen3-30B-A3B-Thinking-2507*', cell '71.4 (73.4)' -> reproduced value 71.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "gpqa-diamond",
    "score": 0.667,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'GPQA-D', column 'Qwen3-32B*', cell '66.7 (68.4)' -> reproduced value 66.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "ifeval-unspecified",
    "score": 0.928,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'IFEval', column 'OAI-OSS-20B*', cell '92.8' -> reproduced value 92.8. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.88,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'IFEval', column 'Qwen3-30B-A3B-Thinking-2507*', cell '88 (88.9)' -> reproduced value 88. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "ifeval-unspecified",
    "score": 0.884,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'IFEval', column 'Qwen3-32B*', cell '88.4 (85)' -> reproduced value 88.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "korbench",
    "score": 0.723,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'KORBench', column 'OAI-OSS-20B*', cell '72.3' -> reproduced value 72.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "korbench",
    "score": 0.702,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'KORBench', column 'Qwen3-30B-A3B-Thinking-2507*', cell '70.2' -> reproduced value 70.2. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "korbench",
    "score": 0.654,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'KORBench', column 'Qwen3-32B*', cell '65.4' -> reproduced value 65.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmlu",
    "score": 0.817,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU', column 'OAI-OSS-20B*', cell '81.7 (85.3)' -> reproduced value 81.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu",
    "score": 0.869,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU', column 'Qwen3-30B-A3B-Thinking-2507*', cell '86.9' -> reproduced value 86.9. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu",
    "score": 0.862,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMLU', column 'Qwen3-32B*', cell '86.2' -> reproduced value 86.2. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmmlu",
    "score": 0.774,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMMLU', column 'OAI-OSS-20B*', cell '77.4 (75.7)' -> reproduced value 77.4. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmmlu",
    "score": 0.79,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMMLU', column 'Qwen3-30B-A3B-Thinking-2507*', cell '79' -> reproduced value 79. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmmlu",
    "score": 0.79,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'MMMLU', column 'Qwen3-32B*', cell '79 (80.6)' -> reproduced value 79. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "multi-swe-bench",
    "score": 0.095,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'Multi-SWE-Bench', column 'Qwen3-30B-A3B-Thinking-2507*', cell '9.5' -> reproduced value 9.5. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "multi-swe-bench",
    "score": 0.077,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'Multi-SWE-Bench', column 'Qwen3-32B*', cell '7.7' -> reproduced value 7.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "ruler-128k",
    "score": 0.787,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'RULER (128K)', column 'OAI-OSS-20B*', cell '78.7' -> reproduced value 78.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ruler-128k",
    "score": 0.945,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'RULER (128K)', column 'Qwen3-30B-A3B-Thinking-2507*', cell '94.5' -> reproduced value 94.5. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "ruler-128k",
    "score": 0.775,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'RULER (128K)', column 'Qwen3-32B*', cell '77.5' -> reproduced value 77.5. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "simpleqa",
    "score": 0.067,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SimpleQA', column 'OAI-OSS-20B*', cell '6.7' -> reproduced value 6.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "simpleqa",
    "score": 0.236,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SimpleQA', column 'Qwen3-30B-A3B-Thinking-2507*', cell '23.6' -> reproduced value 23.6. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "simpleqa",
    "score": 0.086,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SimpleQA', column 'Qwen3-32B*', cell '8.6' -> reproduced value 8.6. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "supergpqa",
    "score": 0.501,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SuperGPQA', column 'OAI-OSS-20B*', cell '50.1' -> reproduced value 50.1. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: OpenAI gpt-oss-20b open-weights checkpoint (label OAI-OSS-20B); reasoning effort not printed. Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.573,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SuperGPQA', column 'Qwen3-30B-A3B-Thinking-2507*', cell '57.3 (56.8)' -> reproduced value 57.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "supergpqa",
    "score": 0.493,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'SuperGPQA', column 'Qwen3-32B*', cell '49.3' -> reproduced value 49.3. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.47,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'TAU1-Airline', column 'Qwen3-30B-A3B-Thinking-2507*', cell '47 (48)' -> reproduced value 47. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-airline",
    "score": 0.38,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'TAU1-Airline', column 'Qwen3-32B*', cell '38' -> reproduced value 38. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.587,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'TAU1-Retail', column 'Qwen3-30B-A3B-Thinking-2507*', cell '58.7 (67.8)' -> reproduced value 58.7. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507 (thinking-only checkpoint, exact label). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-retail",
    "score": 0.409,
    "source": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md",
    "by": "independent",
    "date": "2025-08-20",
    "note": "Seed-OSS-36B-Instruct model card (Apache-2.0, pinned 497f1dc), '### Seed-OSS-36B-Instruct' table, row 'TAU1-Retail', column 'Qwen3-32B*', cell '40.9' -> reproduced value 40.9. ByteDance Seed's own reproduction of a third-party model. Card footnote: '\"*\" indicates that the results in this column are presented in the format of \"reproduced_results (reported_results_if_any)\". Some results have been omitted due to the failure of the evaluation run.' Only the reproduced (unbracketed) value is used; bracketed reported values and bracket-only cells are skipped. Gemma3-27B column skipped ('The results of Gemma3-27B are sourced directly from its technical report'). Checkpoint: Qwen/Qwen3-32B April 2025 hybrid checkpoint; thinking/non-thinking mode not printed (the table compares reasoning models). Same table row as the existing Seed-OSS-36B-Instruct row.",
    "source_id": "card-bytedance-seed--seed-oss-36b-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "bbh",
    "score": 0.631,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Phi-3.5-mini-Ins' = 63.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "bbh",
    "score": 0.554,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Llama-3.2-3B-Ins' = 55.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "bbh",
    "score": 0.562,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Qwen2.5-3B-Ins' = 56.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "bbh",
    "score": 0.724,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Qwen2.5-7B-Ins' = 72.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "bbh",
    "score": 0.533,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Mistral-8B-2410' = 53.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "bbh",
    "score": 0.634,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Llama-3.1-8B-Ins' = 63.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "bbh",
    "score": 0.555,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 55.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "bbh",
    "score": 0.657,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Gemma2-9B-Ins' = 65.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "bbh",
    "score": 0.804,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 80.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mmlu",
    "score": 0.655,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Phi-3.5-mini-Ins' = 65.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu",
    "score": 0.618,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Llama-3.2-3B-Ins' = 61.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "mmlu",
    "score": 0.65,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Qwen2.5-3B-Ins' = 65.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "mmlu",
    "score": 0.726,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Qwen2.5-7B-Ins' = 72.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "mmlu",
    "score": 0.63,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Mistral-8B-2410' = 63.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu",
    "score": 0.681,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Llama-3.1-8B-Ins' = 68.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "mmlu",
    "score": 0.65,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 65.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mmlu",
    "score": 0.713,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Gemma2-9B-Ins' = 71.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu",
    "score": 0.772,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'GPT-4o-mini-2024-07-18' = 77.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mmlu-pro",
    "score": 0.474,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Phi-3.5-mini-Ins' = 47.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu-pro",
    "score": 0.392,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Llama-3.2-3B-Ins' = 39.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "mmlu-pro",
    "score": 0.447,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Qwen2.5-3B-Ins' = 44.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "mmlu-pro",
    "score": 0.562,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Qwen2.5-7B-Ins' = 56.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "mmlu-pro",
    "score": 0.366,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Mistral-8B-2410' = 36.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mmlu-pro",
    "score": 0.44,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Llama-3.1-8B-Ins' = 44.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "mmlu-pro",
    "score": 0.409,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 40.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mmlu-pro",
    "score": 0.501,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Gemma2-9B-Ins' = 50.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mmlu-pro",
    "score": 0.628,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 62.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "arc-challenge",
    "score": 0.846,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Phi-3.5-mini-Ins' = 84.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "arc-challenge",
    "score": 0.761,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Llama-3.2-3B-Ins' = 76.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "arc-challenge",
    "score": 0.826,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Qwen2.5-3B-Ins' = 82.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "arc-challenge",
    "score": 0.901,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Qwen2.5-7B-Ins' = 90.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "arc-challenge",
    "score": 0.827,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Mistral-8B-2410' = 82.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "arc-challenge",
    "score": 0.831,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Llama-3.1-8B-Ins' = 83.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "arc-challenge",
    "score": 0.794,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Llama-3.1-Tulu-3-8B' = 79.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "arc-challenge",
    "score": 0.898,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Gemma2-9B-Ins' = 89.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "arc-challenge",
    "score": 0.935,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'GPT-4o-mini-2024-07-18' = 93.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "boolq",
    "score": 0.777,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Phi-3.5-mini-Ins' = 77.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "boolq",
    "score": 0.714,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Llama-3.2-3B-Ins' = 71.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "boolq",
    "score": 0.654,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Qwen2.5-3B-Ins' = 65.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "boolq",
    "score": 0.8,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Qwen2.5-7B-Ins' = 80.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "boolq",
    "score": 0.805,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Mistral-8B-2410' = 80.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "boolq",
    "score": 0.828,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Llama-3.1-8B-Ins' = 82.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "boolq",
    "score": 0.793,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Llama-3.1-Tulu-3-8B' = 79.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "boolq",
    "score": 0.857,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Gemma2-9B-Ins' = 85.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "boolq",
    "score": 0.887,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'GPT-4o-mini-2024-07-18' = 88.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.266,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Phi-3.5-mini-Ins' = 26.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gpqa-unspecified",
    "score": 0.243,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Llama-3.2-3B-Ins' = 24.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "gpqa-unspecified",
    "score": 0.234,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Qwen2.5-3B-Ins' = 23.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "gpqa-unspecified",
    "score": 0.306,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Qwen2.5-7B-Ins' = 30.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "gpqa-unspecified",
    "score": 0.263,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Mistral-8B-2410' = 26.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gpqa-unspecified",
    "score": 0.263,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Llama-3.1-8B-Ins' = 26.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "gpqa-unspecified",
    "score": 0.299,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 29.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gpqa-unspecified",
    "score": 0.391,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Gemma2-9B-Ins' = 39.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "gpqa-unspecified",
    "score": 0.411,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 41.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "hellaswag",
    "score": 0.722,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Phi-3.5-mini-Ins' = 72.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "hellaswag",
    "score": 0.772,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Llama-3.2-3B-Ins' = 77.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "hellaswag",
    "score": 0.746,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Qwen2.5-3B-Ins' = 74.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "hellaswag",
    "score": 0.8,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Qwen2.5-7B-Ins' = 80.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "hellaswag",
    "score": 0.735,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Mistral-8B-2410' = 73.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "hellaswag",
    "score": 0.728,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Llama-3.1-8B-Ins' = 72.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "hellaswag",
    "score": 0.809,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 80.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "hellaswag",
    "score": 0.871,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Gemma2-9B-Ins' = 87.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "hellaswag",
    "score": 0.887,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'GPT-4o-mini-2024-07-18' = 88.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "openbookqa",
    "score": 0.812,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Phi-3.5-mini-Ins' = 81.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "openbookqa",
    "score": 0.726,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Llama-3.2-3B-Ins' = 72.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "openbookqa",
    "score": 0.793,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Qwen2.5-3B-Ins' = 79.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "openbookqa",
    "score": 0.826,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Qwen2.5-7B-Ins' = 82.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "openbookqa",
    "score": 0.802,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Mistral-8B-2410' = 80.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "openbookqa",
    "score": 0.848,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Llama-3.1-8B-Ins' = 84.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "openbookqa",
    "score": 0.798,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Llama-3.1-Tulu-3-8B' = 79.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "openbookqa",
    "score": 0.9,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Gemma2-9B-Ins' = 90.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "openbookqa",
    "score": 0.9,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'GPT-4o-mini-2024-07-18' = 90.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "piqa",
    "score": 0.782,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Phi-3.5-mini-Ins' = 78.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "piqa",
    "score": 0.682,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Llama-3.2-3B-Ins' = 68.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "piqa",
    "score": 0.726,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Qwen2.5-3B-Ins' = 72.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "piqa",
    "score": 0.762,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Qwen2.5-7B-Ins' = 76.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "piqa",
    "score": 0.812,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Mistral-8B-2410' = 81.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "piqa",
    "score": 0.832,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Llama-3.1-8B-Ins' = 83.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "piqa",
    "score": 0.783,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 78.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "piqa",
    "score": 0.837,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Gemma2-9B-Ins' = 83.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "piqa",
    "score": 0.887,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'GPT-4o-mini-2024-07-18' = 88.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "social-iqa",
    "score": 0.751,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Phi-3.5-mini-Ins' = 75.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "social-iqa",
    "score": 0.683,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Llama-3.2-3B-Ins' = 68.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "social-iqa",
    "score": 0.753,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Qwen2.5-3B-Ins' = 75.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "social-iqa",
    "score": 0.753,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Qwen2.5-7B-Ins' = 75.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "social-iqa",
    "score": 0.776,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Mistral-8B-2410' = 77.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "social-iqa",
    "score": 0.718,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Llama-3.1-8B-Ins' = 71.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "social-iqa",
    "score": 0.734,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 73.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "social-iqa",
    "score": 0.747,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Gemma2-9B-Ins' = 74.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "social-iqa",
    "score": 0.829,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'GPT-4o-mini-2024-07-18' = 82.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "truthfulqa-mc2",
    "score": 0.652,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Phi-3.5-mini-Ins' = 65.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "truthfulqa-mc2",
    "score": 0.592,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Llama-3.2-3B-Ins' = 59.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "truthfulqa-mc2",
    "score": 0.643,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Qwen2.5-3B-Ins' = 64.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.694,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Qwen2.5-7B-Ins' = 69.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "truthfulqa-mc2",
    "score": 0.63,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Mistral-8B-2410' = 63.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "truthfulqa-mc2",
    "score": 0.692,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Llama-3.1-8B-Ins' = 69.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "truthfulqa-mc2",
    "score": 0.641,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Llama-3.1-Tulu-3-8B' = 64.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "truthfulqa-mc2",
    "score": 0.766,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Gemma2-9B-Ins' = 76.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "truthfulqa-mc2",
    "score": 0.782,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'GPT-4o-mini-2024-07-18' = 78.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "winogrande",
    "score": 0.722,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Phi-3.5-mini-Ins' = 72.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "winogrande",
    "score": 0.532,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Llama-3.2-3B-Ins' = 53.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "winogrande",
    "score": 0.633,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Qwen2.5-3B-Ins' = 63.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "winogrande",
    "score": 0.711,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Qwen2.5-7B-Ins' = 71.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "winogrande",
    "score": 0.631,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Mistral-8B-2410' = 63.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "winogrande",
    "score": 0.647,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Llama-3.1-8B-Ins' = 64.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "winogrande",
    "score": 0.654,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 65.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "winogrande",
    "score": 0.74,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Gemma2-9B-Ins' = 74.0. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "winogrande",
    "score": 0.769,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'GPT-4o-mini-2024-07-18' = 76.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "multilingual-mmlu",
    "score": 0.518,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Phi-3.5-mini-Ins' = 51.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "multilingual-mmlu",
    "score": 0.481,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Llama-3.2-3B-Ins' = 48.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "multilingual-mmlu",
    "score": 0.559,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Qwen2.5-3B-Ins' = 55.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "multilingual-mmlu",
    "score": 0.644,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Qwen2.5-7B-Ins' = 64.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "multilingual-mmlu",
    "score": 0.537,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Mistral-8B-2410' = 53.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "multilingual-mmlu",
    "score": 0.562,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Llama-3.1-8B-Ins' = 56.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "multilingual-mmlu",
    "score": 0.545,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Llama-3.1-Tulu-3-8B' = 54.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "multilingual-mmlu",
    "score": 0.638,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Gemma2-9B-Ins' = 63.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "multilingual-mmlu",
    "score": 0.729,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'GPT-4o-mini-2024-07-18' = 72.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mgsm",
    "score": 0.496,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Phi-3.5-mini-Ins' = 49.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mgsm",
    "score": 0.446,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Llama-3.2-3B-Ins' = 44.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "mgsm",
    "score": 0.535,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Qwen2.5-3B-Ins' = 53.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "mgsm",
    "score": 0.645,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Qwen2.5-7B-Ins' = 64.5. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "mgsm",
    "score": 0.567,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Mistral-8B-2410' = 56.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "mgsm",
    "score": 0.567,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Llama-3.1-8B-Ins' = 56.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "mgsm",
    "score": 0.586,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 58.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "mgsm",
    "score": 0.751,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Gemma2-9B-Ins' = 75.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mgsm",
    "score": 0.817,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 81.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "gsm8k",
    "score": 0.769,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Phi-3.5-mini-Ins' = 76.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gsm8k",
    "score": 0.756,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Llama-3.2-3B-Ins' = 75.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "gsm8k",
    "score": 0.806,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Qwen2.5-3B-Ins' = 80.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "gsm8k",
    "score": 0.887,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Qwen2.5-7B-Ins' = 88.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "gsm8k",
    "score": 0.819,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Mistral-8B-2410' = 81.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "gsm8k",
    "score": 0.824,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Llama-3.1-8B-Ins' = 82.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "gsm8k",
    "score": 0.843,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 84.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "gsm8k",
    "score": 0.849,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Gemma2-9B-Ins' = 84.9. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "gsm8k",
    "score": 0.913,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 91.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "math",
    "score": 0.498,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Phi-3.5-mini-Ins' = 49.8. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: microsoft/Phi-3.5-mini-instruct (label 'Phi-3.5-mini-Ins'; same lab). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "math",
    "score": 0.467,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Llama-3.2-3B-Ins' = 46.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.2-3B-Instruct (label 'Llama-3.2-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "math",
    "score": 0.617,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Qwen2.5-3B-Ins' = 61.7. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-3B-Instruct (label 'Qwen2.5-3B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "math",
    "score": 0.604,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Qwen2.5-7B-Ins' = 60.4. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Qwen/Qwen2.5-7B-Instruct (label 'Qwen2.5-7B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-8b-2410",
    "bench": "math",
    "score": 0.416,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Mistral-8B-2410' = 41.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: Ministral-8B-Instruct-2410 / API ministral-8b-2410 (label 'Mistral-8B-2410'; Mistral's only 8B model dated 2410). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "math",
    "score": 0.476,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Llama-3.1-8B-Ins' = 47.6. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: meta-llama/Llama-3.1-8B-Instruct (label 'Llama-3.1-8B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "llama-3-1-tulu-3-8b",
    "bench": "math",
    "score": 0.461,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Llama-3.1-Tulu-3-8B' = 46.1. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: allenai/Llama-3.1-Tulu-3-8B (exact label 'Llama-3.1-Tulu-3-8B'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gemma-2-9b",
    "bench": "math",
    "score": 0.513,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Gemma2-9B-Ins' = 51.3. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: google/gemma-2-9b-it (label 'Gemma2-9B-Ins'). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "math",
    "score": 0.702,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'GPT-4o-mini-2024-07-18' = 70.2. Microsoft's own run on its internal benchmark platform. Setup quote: 'the 3.8B parameters Phi-4-mini-instruct model was compared with a set of models over a variety of benchmarks using an internal benchmark platform (See Appendix A for benchmark methodology)'; Appendix A: 'In an ideal world, we would never change any prompts in our benchmarks ... this is our default approach, and is the case in the vast majority of models we have run to date ... Few shots will always be the same when comparing different models.' Checkpoint: OpenAI API snapshot gpt-4o-mini-2024-07-18 (exact dated label). Same row and shot setting as the existing Phi-4-mini-instruct row.",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "deepseek-r1",
    "bench": "mmlu-pro",
    "score": 0.84,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'MMLU-Pro (EM)', column 'DeepSeek R1' = 84.0. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1' = original DeepSeek-R1 (January 2025). This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1",
    "bench": "gpqa-diamond",
    "score": 0.715,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'GPQA-Diamond (Pass@1)', column 'DeepSeek R1' = 71.5. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1' = original DeepSeek-R1 (January 2025). This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1",
    "bench": "swe-bench-verified",
    "score": 0.492,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'SWE Verified (Resolved)', column 'DeepSeek R1' = 49.2. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1' = original DeepSeek-R1 (January 2025). This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aider-polyglot",
    "score": 0.716,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'Aider-Polyglot (Acc.)', column 'DeepSeek R1 0528' = 71.6. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1 0528' = DeepSeek-R1-0528. This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2024",
    "score": 0.914,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'AIME 2024 (Pass@1)', column 'DeepSeek R1 0528' = 91.4. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1 0528' = DeepSeek-R1-0528. This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.794,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'HMMT 2025 (Pass@1)', column 'DeepSeek R1 0528' = 79.4. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1 0528' = DeepSeek-R1-0528. This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.733,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'LiveCodeBench (2408-2505) (Pass@1)', column 'DeepSeek R1 0528' = 73.3. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1 0528' = DeepSeek-R1-0528. This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "mmlu-redux",
    "score": 0.934,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/blob/4236a6af538feda4548eca9ab308586007567f52/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528 model card (MIT, pinned 4236a6a), '### DeepSeek-R1-0528' table, row 'MMLU-Redux (EM)', column 'DeepSeek R1 0528' = 93.4. Same-lab column of DeepSeek's own table (no cell marked as copied). Setup: 'For all our models, the maximum generation length is set to 64K tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Note under table: 'We use Agentless framework to evaluate model performance on SWE-Verified. We only evaluate text-only prompts in HLE testsets.' Checkpoint: 'DeepSeek R1 0528' = DeepSeek-R1-0528. This cell was not imported with the original rows although its partner cell in the same row was.",
    "source_id": "card-deepseek-ai--deepseek-r1-0528"
  },
  {
    "model": "qwen3-8b",
    "bench": "tau2-telecom",
    "score": 0.278,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "arXiv 2601.02346v1 (CC BY 4.0) Table 5 'Code generation with reasoning evaluations', row 'Qwen3-8B', column 'τ2-Telecom' = 27.8. Setup (Sec. 4.1): 'For τ2-Bench Telecom, each task is limited to 100 steps, and the Qwen3 235B A22B 2507 (Non-reasoning) model acts as the user agent simulator ... For TB Hard, τ2-Bench Telecom, and SciCode, we repeat each evaluation three times and report the average result'; Table 3 settings: 3 responses, 32768 max tokens. Baselines: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations.' Cells marked ∗ ('Results marked with ∗ are taken from Artificial Analysis': Apriel-1.5-15b-Thinker, GPT-OSS-20B) are not used. Qwen/Qwen3-8B (April 2025 hybrid); thinking toggle not stated, baselines run as reasoning models with each baseline's recommended configuration (same as the existing Qwen3-8B rows of this paper). Non-standard user simulator (Qwen3-235B-A22B-2507 instead of the usual GPT-4.1), kept as this paper's own run.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-telecom",
    "score": 0.298,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "arXiv 2601.02346v1 (CC BY 4.0) Table 5 'Code generation with reasoning evaluations', row 'Qwen3-32B', column 'τ2-Telecom' = 29.8. Setup (Sec. 4.1): 'For τ2-Bench Telecom, each task is limited to 100 steps, and the Qwen3 235B A22B 2507 (Non-reasoning) model acts as the user agent simulator ... For TB Hard, τ2-Bench Telecom, and SciCode, we repeat each evaluation three times and report the average result'; Table 3 settings: 3 responses, 32768 max tokens. Baselines: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations.' Cells marked ∗ ('Results marked with ∗ are taken from Artificial Analysis': Apriel-1.5-15b-Thinker, GPT-OSS-20B) are not used. Qwen/Qwen3-32B (April 2025 hybrid); thinking toggle not stated, as for the existing Qwen3-32B rows of this paper. Non-standard user simulator (Qwen3-235B-A22B-2507 instead of the usual GPT-4.1), kept as this paper's own run.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "tau2-telecom",
    "score": 0.254,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "lab",
    "date": "2026-01-05",
    "note": "arXiv 2601.02346v1 (CC BY 4.0) Table 5 'Code generation with reasoning evaluations', row 'Falcon-H1R-7B', column 'τ2-Telecom' = 25.4. Setup (Sec. 4.1): 'For τ2-Bench Telecom, each task is limited to 100 steps, and the Qwen3 235B A22B 2507 (Non-reasoning) model acts as the user agent simulator ... For TB Hard, τ2-Bench Telecom, and SciCode, we repeat each evaluation three times and report the average result'; Table 3 settings: 3 responses, 32768 max tokens. Baselines: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations.' Cells marked ∗ ('Results marked with ∗ are taken from Artificial Analysis': Apriel-1.5-15b-Thinker, GPT-OSS-20B) are not used. Own model (TII). Non-standard user simulator (Qwen3-235B-A22B-2507 instead of the usual GPT-4.1), kept as this paper's own run.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "aime-2025",
    "score": 0.307,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'AIME 2025', column 'Qwen3-1.7B' = 30.7. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "aime-2025",
    "score": 0.588,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'AIME 2025', column 'Qwen3-4B' = 58.8. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bfcl-version-unspecified",
    "score": 0.888,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'BFCL', column 'Qwen3-1.7B' = 88.8. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "bfcl-version-unspecified",
    "score": 0.955,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'BFCL', column 'Qwen3-4B' = 95.5. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "global-mmlu",
    "score": 0.623,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'Global MMLU', column 'Qwen3-1.7B' = 62.3. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "global-mmlu",
    "score": 0.733,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'Global MMLU', column 'Qwen3-4B' = 73.3. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-diamond",
    "score": 0.399,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'GPQA Diamond', column 'Qwen3-1.7B' = 39.9. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "gpqa-diamond",
    "score": 0.553,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'GPQA Diamond', column 'Qwen3-4B' = 55.3. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gsm-plus",
    "score": 0.794,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'GSM-Plus', column 'Qwen3-1.7B' = 79.4. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "gsm-plus",
    "score": 0.882,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'GSM-Plus', column 'Qwen3-4B' = 88.2. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifeval-unspecified",
    "score": 0.742,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'IFEval', column 'Qwen3-1.7B' = 74.2. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "ifeval-unspecified",
    "score": 0.854,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'IFEval', column 'Qwen3-4B' = 85.4. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "livecodebench-v4",
    "score": 0.344,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'LiveCodeBench v4', column 'Qwen3-1.7B' = 34.4. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "livecodebench-v4",
    "score": 0.529,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'LiveCodeBench v4', column 'Qwen3-4B' = 52.9. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mixeval-hard",
    "score": 0.339,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'MixEval Hard', column 'Qwen3-1.7B' = 33.9. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "qwen3-4b",
    "bench": "mixeval-hard",
    "score": 0.38,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md#evaluation",
    "by": "independent",
    "date": "2025-09-10",
    "note": "SmolLM3-3B model card (Apache-2.0, pinned a07cc9a), '#### Extended Thinking' table, row 'MixEval Hard', column 'Qwen3-4B' = 38.0. Hugging Face's own run: '## Evaluation ... All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them.' Checkpoint: Qwen/Qwen3-4B (April 2025 hybrid) in thinking mode (table: 'Evaluation results in reasoning mode for SmolLM3 and Qwen3 models'). BFCL version and IFEval aggregation not stated (same as the existing SmolLM3 row). Same table row as the existing SmolLM3-3B extended-thinking row.",
    "source_id": "card2-huggingfacetb-smollm3-3b"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "omni-math",
    "score": 0.766,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning', column 'OmniMath', cell '76.6 (0.5)' -> 76.6. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: own model.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2024",
    "score": 0.813,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'AIME 24', cell '81.3 (1.8)' -> 81.3. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2025",
    "score": 0.78,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'AIME 25', cell '78.0 (4.6)' -> 78.0. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "hmmt-feb-2025",
    "score": 0.536,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'HMMT Feb 2025', cell '53.6 (6.3)' -> 53.6. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "omni-math",
    "score": 0.819,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'OmniMath', cell '81.9 (0.1)' -> 81.9. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "gpqa-diamond",
    "score": 0.693,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'GPQA-D', cell '69.3 (2.1)' -> 69.3. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "livecodebench-202408-202501",
    "score": 0.531,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Phi-4-reasoning-plus', column 'LCB 8/24-1/25', cell '53.1' -> 53.1. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). LiveCodeBench for the Phi models has no std in the table; Sec. 5.1: 'Our numbers were within 1 accuracy point for LiveCodeBench ... as reported in [21]' (Microsoft ran its own models; only baselines were copied). Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "aime-2024",
    "score": 0.693,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1-Distill-70B', column 'AIME 24', cell '69.3 (2.7)' -> 69.3. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-70B.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "aime-2025",
    "score": 0.515,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1-Distill-70B', column 'AIME 25', cell '51.5 (5.8)' -> 51.5. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-70B.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "omni-math",
    "score": 0.634,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1-Distill-70B', column 'OmniMath', cell '63.4 (0.4)' -> 63.4. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-70B.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1-distill-llama-70b",
    "bench": "gpqa-diamond",
    "score": 0.662,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1-Distill-70B', column 'GPQA-D', cell '66.2 (2.4)' -> 66.2. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-70B.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2024",
    "score": 0.787,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1', column 'AIME 24', cell '78.7 (3.8)' -> 78.7. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: original DeepSeek-R1 (January 2025; paper dated April 2025, before R1-0528).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2025",
    "score": 0.704,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1', column 'AIME 25', cell '70.4 (4.3)' -> 70.4. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: original DeepSeek-R1 (January 2025; paper dated April 2025, before R1-0528).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1",
    "bench": "omni-math",
    "score": 0.85,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1', column 'OmniMath', cell '85.0 (0.6)' -> 85.0. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: original DeepSeek-R1 (January 2025; paper dated April 2025, before R1-0528).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "deepseek-r1",
    "bench": "gpqa-diamond",
    "score": 0.73,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'DeepSeek-R1', column 'GPQA-D', cell '73.0 (1.7)' -> 73.0. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: original DeepSeek-R1 (January 2025; paper dated April 2025, before R1-0528).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o1-mini",
    "bench": "hmmt-feb-2025",
    "score": 0.38,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o1-mini', column 'HMMT Feb 2025', cell '38.0 (6.2)' -> 38.0. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI o1-mini (only snapshot o1-mini-2024-09-12).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "aime-2024",
    "score": 0.746,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o1', column 'AIME 24', cell '74.6 (6.5)' -> 74.6. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI 'o1' in an April 2025 paper = the only GA o1 snapshot o1-2024-12-17; reasoning effort not printed.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "aime-2025",
    "score": 0.714,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o1', column 'AIME 25', cell '71.4 (5.7)' -> 71.4. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI 'o1' in an April 2025 paper = the only GA o1 snapshot o1-2024-12-17; reasoning effort not printed.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "omni-math",
    "score": 0.675,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o1', column 'OmniMath', cell '67.5 (0.9)' -> 67.5. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI 'o1' in an April 2025 paper = the only GA o1 snapshot o1-2024-12-17; reasoning effort not printed.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "gpqa-diamond",
    "score": 0.767,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o1', column 'GPQA-D', cell '76.7 (1.8)' -> 76.7. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI 'o1' in an April 2025 paper = the only GA o1 snapshot o1-2024-12-17; reasoning effort not printed.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "aime-2024",
    "score": 0.88,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o3-mini-high', column 'AIME 24', cell '88.0 (5.5)' -> 88.0. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI o3-mini (only snapshot o3-mini-2025-01-31) at reasoning effort high (label 'o3-mini-high').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "aime-2025",
    "score": 0.825,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o3-mini-high', column 'AIME 25', cell '82.5 (4.9)' -> 82.5. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI o3-mini (only snapshot o3-mini-2025-01-31) at reasoning effort high (label 'o3-mini-high').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "omni-math",
    "score": 0.746,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o3-mini-high', column 'OmniMath', cell '74.6 (5.1)' -> 74.6. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI o3-mini (only snapshot o3-mini-2025-01-31) at reasoning effort high (label 'o3-mini-high').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "gpqa-diamond",
    "score": 0.777,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'o3-mini-high', column 'GPQA-D', cell '77.7 (0.6)' -> 77.7. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: OpenAI o3-mini (only snapshot o3-mini-2025-01-31) at reasoning effort high (label 'o3-mini-high').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "aime-2024",
    "score": 0.553,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Claude-3.7-Sonnet', column 'AIME 24', cell '55.3 (3.0)' -> 55.3. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: Claude 3.7 Sonnet (only snapshot claude-3-7-sonnet-20250219); caption: 'Claude-3.7-Sonnet is evaluated with thinking enabled' (budget not printed).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "aime-2025",
    "score": 0.53,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Claude-3.7-Sonnet', column 'AIME 25', cell '53 (5.8)' -> 53. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: Claude 3.7 Sonnet (only snapshot claude-3-7-sonnet-20250219); caption: 'Claude-3.7-Sonnet is evaluated with thinking enabled' (budget not printed).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "omni-math",
    "score": 0.546,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Claude-3.7-Sonnet', column 'OmniMath', cell '54.6 (0.9)' -> 54.6. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: Claude 3.7 Sonnet (only snapshot claude-3-7-sonnet-20250219); caption: 'Claude-3.7-Sonnet is evaluated with thinking enabled' (budget not printed).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "gpqa-diamond",
    "score": 0.768,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 1 'Average Pass@1 accuracy (%) of models on selected reasoning benchmarks', row 'Claude-3.7-Sonnet', column 'GPQA-D', cell '76.8 (1.3)' -> 76.8. Caption: 'We report standard deviation in parentheses for all results we produced using Eureka [10] and MathArena's [11] judge/scoring function for HMMT'; Sec. 4: 'For all these results, we use the implementation pipelines provided in the Eureka ML Insights repository to evaluate all models ... we reuse the same evaluation logs for baseline comparison'. Only cells carrying a standard deviation (own runs) are used; cells without one are copied per the caption ('Scores for all baselines for LiveCodeBench are from [21] or LiveCodeBench leaderboard. Sores for HMMT 2025 are from MathArena leaderboards. All other scores for OpenThinker2, QwQ, EXAONE-Deep and Gemini-2.5-Pro are reproduced from their corresponding reports'). Checkpoint: Claude 3.7 Sonnet (only snapshot claude-3-7-sonnet-20250219); caption: 'Claude-3.7-Sonnet is evaluated with thinking enabled' (budget not printed).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4",
    "bench": "flenqa-3k",
    "score": 0.82,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'FlenQA [3K-token subset] [30]', column 'Phi-4' = 82.0. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/phi-4 (same lab). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "flenqa-3k",
    "score": 0.977,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'FlenQA [3K-token subset] [30]', column 'Phi-4-reasoning' = 97.7. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: own model. New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "flenqa-3k",
    "score": 0.979,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'FlenQA [3K-token subset] [30]', column 'Phi-4-reasoning-plus' = 97.9. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4",
    "bench": "ifeval-unspecified",
    "score": 0.623,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'IFEval Strict [62]', column 'Phi-4' = 62.3. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/phi-4 (same lab). IFEval label 'IFEval Strict' (prompt/instruction level not stated), same row as the existing Phi-4-reasoning row.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "ifeval-unspecified",
    "score": 0.849,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'IFEval Strict [62]', column 'Phi-4-reasoning-plus' = 84.9. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry). IFEval label 'IFEval Strict' (prompt/instruction level not stated), same row as the existing Phi-4-reasoning row.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "ifeval-unspecified",
    "score": 0.915,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'IFEval Strict [62]', column 'o3-mini' = 91.5. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: OpenAI o3-mini (only snapshot o3-mini-2025-01-31); reasoning effort not printed in Table 2. IFEval label 'IFEval Strict' (prompt/instruction level not stated), same row as the existing Phi-4-reasoning row.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4",
    "bench": "humaneval-plus",
    "score": 0.835,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'HumanEvalPlus [35]', column 'Phi-4' = 83.5. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/phi-4 (same lab). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "humaneval-plus",
    "score": 0.929,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'HumanEvalPlus [35]', column 'Phi-4-reasoning' = 92.9. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: own model. New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "humaneval-plus",
    "score": 0.923,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'HumanEvalPlus [35]', column 'Phi-4-reasoning-plus' = 92.3. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4",
    "bench": "mmlu-pro",
    "score": 0.715,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'MMLUPro [57]', column 'Phi-4' = 71.5. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/phi-4 (same lab). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "mmlu-pro",
    "score": 0.743,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'MMLUPro [57]', column 'Phi-4-reasoning' = 74.3. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: own model. New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "mmlu-pro",
    "score": 0.76,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (arXiv 2504.21318v1, CC BY 4.0) Table 2, row 'MMLUPro [57]', column 'Phi-4-reasoning-plus' = 76.0. Table 2 caption: 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations. For the results in this table Phi-4-reasoning and Phi-4-reasoning-plus were evaluated at temperature 0.8, while Phi-4 was evaluated at temperature of 0.0.' Sec. 5: 'we evaluate our models on MMLU-pro [57], HumanEvalPlus [35], and ArenaHard [33]. Next, we also use a set of general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA [30], Toxigen [23], Kitab [3] and IFEval [62]' with footnote '... except IFEval ..., for which we ran the evaluation three times (for all models) and report the average accuracy.' Checkpoint: microsoft/Phi-4-reasoning-plus (same lab; proposed comparison-only entry). New run for this paper (own-model rows for Phi models only; o3-mini/GPT-4o cells not used because the own-evaluation statement for this row names 'our models').",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2024",
    "score": 0.772,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 4 (Mathematical reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'AIME24', cell '77.2'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "aime-2025",
    "score": 0.712,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 4 (Mathematical reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'AIME25', cell '71.2'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.477,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 4 (Mathematical reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'HMMT25', cell '47.7'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "math-500",
    "score": 0.954,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 4 (Mathematical reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'MATH500', cell '95.4'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.531,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 5 (Code generation with reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'LCB v6', cell '53.1'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "scicode-main",
    "score": 0.072,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 5 (Code generation with reasoning evaluations), row 'Phi-4-Reasoning-Plus-14B', column 'SciCode sub / main', cell '29.8 / 7.2' -> main-problem value only (subproblem value not imported, as for the existing rows). TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "gpqa-diamond",
    "score": 0.679,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 6 (General domains reasoning), row 'Phi-4-Reasoning-Plus-14B', column 'GPQA-Dimond', cell '67.9'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "mmlu-pro",
    "score": 0.792,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 6 (General domains reasoning), row 'Phi-4-Reasoning-Plus-14B', column 'MMLU-Pro', cell '79.2'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "hle",
    "score": 0.059,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 6 (General domains reasoning), row 'Phi-4-Reasoning-Plus-14B', column 'HLE', cell '5.9'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "ifbench-unspecified",
    "score": 0.517,
    "source": "https://arxiv.org/html/2601.02346v1",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R paper (arXiv 2601.02346v1, CC BY 4.0) Table 6 (General domains reasoning), row 'Phi-4-Reasoning-Plus-14B', column 'IFBench', cell '51.7'. TII's own baseline run, Sec. 4.1: 'To compare performance, we evaluated a range of state-of-the-art (SOTA) models across 7B-32B parameter ranges, using each baseline's recommended evaluation configurations ... In the mid-size range, we evaluated Phi-4-Reasoning-Plus-14B'. Not a ∗ (Artificial Analysis) cell. Checkpoint: microsoft/Phi-4-reasoning-plus (proposed comparison-only entry). Same table row/settings as the existing Falcon-H1R-7B and baseline rows of this paper.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-pro",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'MMLU-Pro', column 'Qwen3.5-27B' = 86.1. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-27B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'MMLU-Pro', column 'Qwen3.5-35B-A3B' = 85.3. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-35B-A3B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "gpqa-diamond",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'GPQA Diamond', column 'Qwen3.5-27B' = 85.5. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-27B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.842,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'GPQA Diamond', column 'Qwen3.5-35B-A3B' = 84.2. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-35B-A3B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hle",
    "score": 0.243,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'HLE w/ CoT', column 'Qwen3.5-27B' = 24.3. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-27B (thinking default). Same row as the existing Qwen3.5-122B-A10B row (HLE w/ CoT, no tools).",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hle",
    "score": 0.224,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'HLE w/ CoT', column 'Qwen3.5-35B-A3B' = 22.4. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-35B-A3B (thinking default). Same row as the existing Qwen3.5-122B-A10B row (HLE w/ CoT, no tools).",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-verified",
    "score": 0.724,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'SWE-bench Verified', column 'Qwen3.5-27B' = 72.4. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-27B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.692,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'SWE-bench Verified', column 'Qwen3.5-35B-A3B' = 69.2. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-35B-A3B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "livecodebench-v6",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'LiveCodeBench v6', column 'Qwen3.5-27B' = 80.7. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-27B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'LiveCodeBench v6', column 'Qwen3.5-35B-A3B' = 74.6. Same-lab Qwen3.5 column in Qwen's own table (no cell marked as copied); other rows of these two columns are already recorded from this card. Card footnotes under '### Language' (e.g. 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card'); Qwen3.5 models run in thinking mode by default (table does not label mode). Checkpoint: Qwen/Qwen3.5-35B-A3B (thinking default). Same row as the existing Qwen3.5-122B-A10B row.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-bench",
    "score": 0.698,
    "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md",
    "by": "independent",
    "date": "2026-02-24",
    "note": "Qwen3.5-122B-A10B model card (Apache-2.0, pinned dc4d348), '### Language' table, row 'TAU2-Bench', column 'GPT-5-mini 2025-08-07' = 69.8. Qwen's own run: footnote 'TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.' Only this row carries an all-models evaluation statement, so the other GPT-5-mini/GPT-OSS-120B cells of the table are not used. Checkpoint: column header 'GPT-5-mini 2025-08-07' = OpenAI gpt-5-mini-2025-08-07; reasoning effort not printed.",
    "source_id": "card-qwen--qwen3.5-122b-a10b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-verified",
    "score": 0.75,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'SWE-bench Verified', column 'Qwen3.5-27B' = 75.0. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "swe-bench-verified",
    "score": 0.762,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'SWE-bench Verified', column 'Qwen3.5-397B-A17B' = 76.2. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-397B-A17B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.734,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'SWE-bench Verified', column 'Qwen3.6-35B-A3B' = 73.4. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.6-35B-A3B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-pro",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'MMLU-Pro', column 'Qwen3.5-27B' = 86.1. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "mmlu-pro",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'MMLU-Pro', column 'Qwen3.5-397B-A17B' = 87.8. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-397B-A17B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.852,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'MMLU-Pro', column 'Qwen3.6-35B-A3B' = 85.2. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.6-35B-A3B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hle",
    "score": 0.243,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'HLE', column 'Qwen3.5-27B' = 24.3. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle",
    "score": 0.287,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'HLE', column 'Qwen3.5-397B-A17B' = 28.7. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-397B-A17B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hle",
    "score": 0.214,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'HLE', column 'Qwen3.6-35B-A3B' = 21.4. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.6-35B-A3B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "livecodebench-v6",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'LiveCodeBench v6', column 'Qwen3.5-27B' = 80.7. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "livecodebench-v6",
    "score": 0.836,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'LiveCodeBench v6', column 'Qwen3.5-397B-A17B' = 83.6. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-397B-A17B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.804,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'LiveCodeBench v6', column 'Qwen3.6-35B-A3B' = 80.4. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.6-35B-A3B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "androidworld",
    "score": 0.642,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Vision Language' table, row 'AndroidWorld', column 'Qwen3.5-27B' = 64.2. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "gpqa-diamond",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'GPQA Diamond', column 'Qwen3.5-27B' = 85.5. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-27B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "gpqa-diamond",
    "score": 0.884,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "lab",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'GPQA Diamond', column 'Qwen3.5-397B-A17B' = 88.4. Same-lab column in Qwen's own table (no cell marked as copied; other rows of this column are already recorded from this card). Card footnote 'SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'; Qwen3.6/3.5 models run in thinking mode by default. Checkpoint: Qwen/Qwen3.5-397B-A17B. Same row as the existing Qwen3.6-27B row. Note: the Qwen3.5 values on the knowledge/reasoning rows equal those printed in the Qwen3.5 cards (carried over from Qwen's earlier own run).",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.357,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'SWE-bench Pro', column 'Gemma4-31B' = 35.7. Qwen's own run of a third-party model: footnote 'We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.' Other rows of this column carry no such statement and are not used. Checkpoint: google/gemma-4-31B-it (label 'Gemma4-31B'); thinking toggle not printed.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.571,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'SWE-bench Pro', column 'Claude 4.5 Opus' = 57.1. Qwen's own run of a third-party model: footnote 'We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.' Other rows of this column carry no such statement and are not used. Checkpoint: Anthropic Claude Opus 4.5 (label 'Claude 4.5 Opus'); effort/thinking not printed.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "nl2repo",
    "score": 0.155,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'NL2Repo', column 'Gemma4-31B' = 15.5. Qwen's own run of a third-party model: footnote 'NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).' Other rows of this column carry no such statement and are not used. Checkpoint: google/gemma-4-31B-it (label 'Gemma4-31B'); thinking toggle not printed.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "nl2repo",
    "score": 0.432,
    "source": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md",
    "by": "independent",
    "date": "2026-04-22",
    "note": "Qwen3.6-27B model card (Apache-2.0, pinned 6a9e13b), '### Language' table, row 'NL2Repo', column 'Claude 4.5 Opus' = 43.2. Qwen's own run of a third-party model: footnote 'NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).' Other rows of this column carry no such statement and are not used. Checkpoint: Anthropic Claude Opus 4.5 (label 'Claude 4.5 Opus'); effort/thinking not printed.",
    "source_id": "card-qwen--qwen3.6-27b"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-pro",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-235B-A22B Thinking' = 82.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.785,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-30B-A3B Thinking' = 78.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-redux",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-Redux', column 'Qwen3-235B-A22B Thinking' = 92.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-redux",
    "score": 0.895,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-Redux', column 'Qwen3-30B-A3B Thinking' = 89.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.914,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-Redux', column 'Qwen3-30B-A3B-Thinking-2507' = 91.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "gpqa-unspecified",
    "score": 0.711,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'GPQA', column 'Qwen3-235B-A22B Thinking' = 71.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "gpqa-unspecified",
    "score": 0.658,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'GPQA', column 'Qwen3-30B-A3B Thinking' = 65.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "gpqa-unspecified",
    "score": 0.734,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'GPQA', column 'Qwen3-30B-A3B-Thinking-2507' = 73.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "supergpqa",
    "score": 0.607,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'SuperGPQA', column 'Qwen3-235B-A22B Thinking' = 60.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "supergpqa",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'SuperGPQA', column 'Qwen3-30B-A3B Thinking' = 51.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "supergpqa",
    "score": 0.568,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'SuperGPQA', column 'Qwen3-30B-A3B-Thinking-2507' = 56.8. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aime-2025",
    "score": 0.815,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'AIME25', column 'Qwen3-235B-A22B Thinking' = 81.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aime-2025",
    "score": 0.709,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'AIME25', column 'Qwen3-30B-A3B Thinking' = 70.9. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.625,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'HMMT25', column 'Qwen3-235B-A22B Thinking' = 62.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.498,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'HMMT25', column 'Qwen3-30B-A3B Thinking' = 49.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'HMMT25', column 'Qwen3-30B-A3B-Thinking-2507' = 71.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livebench-20241125",
    "score": 0.771,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'LiveBench 20241125', column 'Qwen3-235B-A22B Thinking' = 77.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livebench-20241125",
    "score": 0.743,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'LiveBench 20241125', column 'Qwen3-30B-A3B Thinking' = 74.3. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "livebench-20241125",
    "score": 0.768,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'LiveBench 20241125', column 'Qwen3-30B-A3B-Thinking-2507' = 76.8. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livecodebench-v6",
    "score": 0.557,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-235B-A22B Thinking' = 55.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.574,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-30B-A3B Thinking' = 57.4. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "ojbench",
    "score": 0.256,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'OJBench', column 'Qwen3-235B-A22B Thinking' = 25.6. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ojbench",
    "score": 0.207,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'OJBench', column 'Qwen3-30B-A3B Thinking' = 20.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ojbench",
    "score": 0.251,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'OJBench', column 'Qwen3-30B-A3B-Thinking-2507' = 25.1. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "ifeval-unspecified",
    "score": 0.834,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'IFEval', column 'Qwen3-235B-A22B Thinking' = 83.4. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.865,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'IFEval', column 'Qwen3-30B-A3B Thinking' = 86.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "ifeval-unspecified",
    "score": 0.889,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'IFEval', column 'Qwen3-30B-A3B-Thinking-2507' = 88.9. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "arena-hard-v2",
    "score": 0.615,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'Arena-Hard v2$', column 'Qwen3-235B-A22B Thinking' = 61.5. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.363,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'Arena-Hard v2$', column 'Qwen3-30B-A3B Thinking' = 36.3. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "arena-hard-v2",
    "score": 0.56,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'Arena-Hard v2$', column 'Qwen3-30B-A3B-Thinking-2507' = 56.0. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "creative-writing-v3",
    "score": 0.846,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'Creative Writing v3', column 'Qwen3-235B-A22B Thinking' = 84.6. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "creative-writing-v3",
    "score": 0.791,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'Creative Writing v3', column 'Qwen3-30B-A3B Thinking' = 79.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "writingbench",
    "score": 0.803,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'WritingBench', column 'Qwen3-235B-A22B Thinking' = 80.3. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "writingbench",
    "score": 0.77,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'WritingBench', column 'Qwen3-30B-A3B Thinking' = 77.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "writingbench",
    "score": 0.85,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'WritingBench', column 'Qwen3-30B-A3B-Thinking-2507' = 85.0. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "bfcl-v3",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'BFCL-v3', column 'Qwen3-235B-A22B Thinking' = 70.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'BFCL-v3', column 'Qwen3-30B-A3B Thinking' = 69.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.724,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'BFCL-v3', column 'Qwen3-30B-A3B-Thinking-2507' = 72.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-retail",
    "score": 0.548,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Retail', column 'Qwen3-235B-A22B Thinking' = 54.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-retail",
    "score": 0.617,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Retail', column 'Qwen3-30B-A3B Thinking' = 61.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-retail",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Retail', column 'Qwen3-30B-A3B-Thinking-2507' = 67.8. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-airline",
    "score": 0.26,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Airline', column 'Qwen3-235B-A22B Thinking' = 26.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Airline', column 'Qwen3-30B-A3B Thinking' = 32.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau-bench-airline",
    "score": 0.48,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU1-Airline', column 'Qwen3-30B-A3B-Thinking-2507' = 48.0. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-retail",
    "score": 0.404,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Retail', column 'Qwen3-235B-A22B Thinking' = 40.4. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-retail",
    "score": 0.342,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Retail', column 'Qwen3-30B-A3B Thinking' = 34.2. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.588,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Retail', column 'Qwen3-30B-A3B-Thinking-2507' = 58.8. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-airline",
    "score": 0.3,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Airline', column 'Qwen3-235B-A22B Thinking' = 30.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-airline",
    "score": 0.36,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Airline', column 'Qwen3-30B-A3B Thinking' = 36.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Airline', column 'Qwen3-30B-A3B-Thinking-2507' = 58.0. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.219,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Telecom', column 'Qwen3-235B-A22B Thinking' = 21.9. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.228,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Telecom', column 'Qwen3-30B-A3B Thinking' = 22.8. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.263,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'TAU2-Telecom', column 'Qwen3-30B-A3B-Thinking-2507' = 26.3. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "multi-if-unspecified",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MultiIF', column 'Qwen3-235B-A22B Thinking' = 71.9. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multi-if-unspecified",
    "score": 0.722,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MultiIF', column 'Qwen3-30B-A3B Thinking' = 72.2. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "multi-if-unspecified",
    "score": 0.764,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MultiIF', column 'Qwen3-30B-A3B-Thinking-2507' = 76.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-prox",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-ProX', column 'Qwen3-235B-A22B Thinking' = 80.0. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.731,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-ProX', column 'Qwen3-30B-A3B Thinking' = 73.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.764,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'MMLU-ProX', column 'Qwen3-30B-A3B-Thinking-2507' = 76.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "include",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'INCLUDE', column 'Qwen3-235B-A22B Thinking' = 78.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "include",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'INCLUDE', column 'Qwen3-30B-A3B Thinking' = 71.9. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "include",
    "score": 0.744,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'INCLUDE', column 'Qwen3-30B-A3B-Thinking-2507' = 74.4. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "polymath",
    "score": 0.547,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'PolyMATH', column 'Qwen3-235B-A22B Thinking' = 54.7. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-235B-A22B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "polymath",
    "score": 0.461,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'PolyMATH', column 'Qwen3-30B-A3B Thinking' = 46.1. Same-lab comparison column in Qwen's own table (no cell marked as copied), as for the 'Qwen3-235B-A22B Thinking' columns already imported from the other Qwen 2507 cards. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: Qwen/Qwen3-30B-A3B (April 2025 hybrid checkpoint) in thinking mode. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "polymath",
    "score": 0.526,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/144afc2f379b542fdd4e85a1fcd5e1f79112d95d/README.md",
    "by": "lab",
    "date": "2025-07-30",
    "note": "Qwen3-30B-A3B-Thinking-2507 model card (Apache-2.0, pinned 144afc2), '## Performance' table, row 'PolyMATH', column 'Qwen3-30B-A3B-Thinking-2507' = 52.6. Own-model cell on a row not imported earlier (only MMLU-Pro, AIME25, LiveCodeBench v6 and Creative Writing v3 were), added so the row's lab comparators have a run partner. Card footnotes: '$ For reproducibility, we report the win rates evaluated by GPT-4.1.' and '& For highly challenging tasks (including PolyMATH and all reasoning and coding tasks), we use an output length of 81,920 tokens. For all other tasks, we set the output length to 32,768.' Checkpoint: own model Qwen/Qwen3-30B-A3B-Thinking-2507. Row-to-benchmark mapping as in the Qwen3-4B-Thinking-2507 card rows.",
    "source_id": "card-qwen--qwen3-30b-a3b-thinking-2507"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2025",
    "score": 0.85,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 14B', column 'AIME25' = 0.850 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-14B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "aime-2024",
    "score": 0.898,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 14B', column 'AIME24' = 0.898 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-14B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.712,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 14B', column 'GPQA Diamond' = 0.712 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-14B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.646,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 14B', column 'LiveCodeBench' = 0.646 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-14B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "aime-2025",
    "score": 0.787,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 8B', column 'AIME25' = 0.787 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-8B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "aime-2024",
    "score": 0.86,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 8B', column 'AIME24' = 0.860 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-8B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.668,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 8B', column 'GPQA Diamond' = 0.668 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-8B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "livecodebench-unversioned",
    "score": 0.616,
    "source": "https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512/blob/4a36357c811bf511a7b625d132e12f22408aac91/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 3B Reasoning 2512 card (Apache-2.0, pinned 4a36357), '## Benchmark Results' / '### Reasoning' table, row 'Ministral 3 8B', column 'LiveCodeBench' = 0.616 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Reasoning section rows are the Reasoning 2512 checkpoints, as in the precedent rows imported from the Ministral 3 8B Reasoning card. Checkpoint: mistralai/Ministral-3-8B-Reasoning-2512. Qwen comparison rows not used (no statement that Mistral ran them). LiveCodeBench version/window not stated.",
    "source_id": "card-mistralai-ministral-3-3b-reasoning-2512"
  },
  {
    "model": "magistral-medium-2506",
    "bench": "aime-2024",
    "score": 0.7359,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.0', column 'AIME24 pass@1' = 73.59%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.0 = API magistral-medium-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2506",
    "bench": "aime-2025",
    "score": 0.6495,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.0', column 'AIME25 pass@1' = 64.95%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.0 = API magistral-medium-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2506",
    "bench": "gpqa-diamond",
    "score": 0.7083,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.0', column 'GPQA Diamond' = 70.83%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.0 = API magistral-medium-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2506",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.5936,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.0', column 'Livecodebench (v5)' = 59.36%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.0 = API magistral-medium-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2506",
    "bench": "aime-2024",
    "score": 0.7068,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.0', column 'AIME24 pass@1' = 70.68%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.0 = mistralai/Magistral-Small-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2506",
    "bench": "aime-2025",
    "score": 0.6276,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.0', column 'AIME25 pass@1' = 62.76%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.0 = mistralai/Magistral-Small-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2506",
    "bench": "gpqa-diamond",
    "score": 0.6818,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.0', column 'GPQA Diamond' = 68.18%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.0 = mistralai/Magistral-Small-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2506",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.5584,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2026-02-23",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.0', column 'Livecodebench (v5)' = 55.84%. Same-lab earlier-version row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.0 = mistralai/Magistral-Small-2506 (version 25.06). Same column as the existing Magistral Small 1.2 row; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-30B-A3B-Instruct-2507' = 78.4. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-30B-A3B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.83,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'MMLU-Pro', column 'Qwen3-235B-A22B-Instruct-2507' = 83.0. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-235B-A22B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.613,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'AIME25', column 'Qwen3-30B-A3B-Instruct-2507' = 61.3. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-30B-A3B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'AIME25', column 'Qwen3-235B-A22B-Instruct-2507' = 70.3. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-235B-A22B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.432,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-30B-A3B-Instruct-2507' = 43.2. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-30B-A3B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livecodebench-v6",
    "score": 0.518,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-Next-80B-A3B-Instruct model card (Apache-2.0, pinned 9c7f2fb), '## Performance' table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-235B-A22B-Instruct-2507' = 51.8. Same-lab column in Qwen's own table (no cell marked as copied); the other rows of this column are already recorded from this card. Non-thinking (instruct-only) checkpoints; footnote '*: For reproducibility, we report the win rates evaluated by GPT-4.1.' Checkpoint: Qwen/Qwen3-235B-A22B-Instruct-2507. Qwen3-32B Non-Thinking column not used (hybrid model recorded in thinking mode only, as for the existing rows).",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2024",
    "score": 0.533,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Qwen-7B' (no asterisk), column 'AIME' = 53.3. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-500",
    "score": 0.914,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Qwen-7B' (no asterisk), column 'MATH-500' = 91.4. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "gpqa-diamond",
    "score": 0.495,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Qwen-7B' (no asterisk), column 'GPQA Diamond' = 49.5. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "aime-2024",
    "score": 0.433,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Llama-8B' (no asterisk), column 'AIME' = 43.3. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "math-500",
    "score": 0.869,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Llama-8B' (no asterisk), column 'MATH-500' = 86.9. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "gpqa-diamond",
    "score": 0.473,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'DeepSeek-R1-Distill-Llama-8B' (no asterisk), column 'GPQA Diamond' = 47.3. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "aime-2024",
    "score": 0.067,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'Llama-3.2-3B-Instruct' (no asterisk), column 'AIME' = 6.7. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: meta-llama/Llama-3.2-3B-Instruct.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "math-500",
    "score": 0.444,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'Llama-3.2-3B-Instruct' (no asterisk), column 'MATH-500' = 44.4. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: meta-llama/Llama-3.2-3B-Instruct.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gpqa-diamond",
    "score": 0.253,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-reasoning model card (MIT, pinned 0e3b1e2), '### Model Quality' table, row 'Llama-3.2-3B-Instruct' (no asterisk), column 'GPQA Diamond' = 25.3. Microsoft's own reproduction. Card: 'the 3.8B parameters Phi-4-mini-reasoning model was compared with a set of models over a variety of reasoning benchmarks'; Appendix A: 'this is our default approach, and is the case in the vast majority of models we have run to date. For all benchmarks, we consider using the same generation configuration such as max sequence length (32768), the same temperature for the fair comparison.' The card's '*' marker is defined in the linked report (arXiv 2504.21233v1 Table 3, same numbers): 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' Only unstarred rows are used; the report is cited only for the marker meaning and the AIME year (AIME 2024), not as a data source. Checkpoint: meta-llama/Llama-3.2-3B-Instruct.",
    "source_id": "card-microsoft-phi-4-mini-reasoning"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.903,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-08-14",
    "note": "Qwen3.8-27B model card (Apache-2.0, pinned 1d4bf0f), '### Text Performance' table, section 'General', row 'Competitive coding / LiveCodeBench v6', column 'Qwen3.8-27B' = 90.3. Own-model cell re-recorded under the benchmark id the existing Qwen3.7-Plus cell of the same row uses (livecodebench-v6-window-unspecified): the card prints 'LiveCodeBench v6' with no date window, which is what that id means. The existing qwen3-8-27b row for this cell sits under livecodebench-v6 (Feb-May 2025) and is alone there; owner: drop or re-bin that row (if re-binned, drop this one). Qwen3.8 runs in thinking mode by default.",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.512,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "independent",
    "date": "2026-08-14",
    "note": "Qwen3.8-27B model card (Apache-2.0, pinned 1d4bf0f), '### Text Performance' table, row 'Agentic coding / SWE-bench Pro', column 'Muse Glimmer-30B' = 51.2. Qwen's own run of a third-party model: footnote 'SWE-bench Pro: Except for Opus4.6 Max, which uses the officially reported score, all models are evaluated with the Claude Code harness at temp=1.0, top_p=0.95, and a 256K context window. Problematic tasks were corrected, and all baseline models were re-evaluated on the refined benchmark.' (Opus4.6 Max cell excluded as officially reported.) Checkpoint: Meta Muse Glimmer 30B (catalogue muse-glimmer-30b, label 'Muse Glimmer-30B'); reasoning setting not printed.",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "glm-5",
    "bench": "browsecomp",
    "score": 0.62,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'BrowseComp', column 'GLM-5' = 62.0. Own GLM-5 cell of the row whose GLM-4.7 cell is already recorded (without context management; the separate 'BrowseComp (w/ Context Manage)' row is not used). Search-agent judging: 'we standardize all judge-based components using the official OpenAI evaluation prompt and the proprietary model o3-mini as the judge'. Thinking/non-thinking mode not labelled.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "glm-4-7",
    "bench": "hmmt-feb-2025",
    "score": 0.971,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'HMMT Feb. 2025', column 'GLM-4.7' = 97.1. Same-lab GLM-4.7 cell of the row whose GLM-5 cell is already recorded; Sec. 6.1.1: 'Most reasoning tasks are evaluated with a maximum generation length of 131,072 tokens'. Mode not labelled.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "glm-5",
    "bench": "mcp-atlas",
    "score": 0.678,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'GLM-5' = 67.8. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: own model.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "glm-4-7",
    "bench": "mcp-atlas",
    "score": 0.52,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "lab",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'GLM-4.7' = 52.0. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: same-lab GLM-4.7.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "mcp-atlas",
    "score": 0.622,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "independent",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'DeepSeek-V3.2' = 62.2. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: deepseek-ai/DeepSeek-V3.2 (label DeepSeek-V3.2); thinking mode not printed.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "kimi-k2-5",
    "bench": "mcp-atlas",
    "score": 0.638,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "independent",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'Kimi K2.5' = 63.8. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: moonshotai/Kimi-K2.5 (label Kimi K2.5); thinking mode not printed.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "mcp-atlas",
    "score": 0.652,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "independent",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'Claude Opus 4.5' = 65.2. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: Anthropic Claude Opus 4.5; effort/thinking not printed.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "gpt-5-2",
    "bench": "mcp-atlas",
    "score": 0.68,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "independent",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (arXiv 2602.15763v2, CC BY 4.0) Table 7 'Comparison between GLM-5 and open-source/proprietary models', row 'MCP-Atlas (Public Set)', column 'GPT-5.2 (xhigh)' = 68.0. Z.ai's own run of every column: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task to avoid task failures due to deployment conditions. We use Gemini 3 Pro as the judge model for MCP-Atlas.' Filed under mcp-atlas (public set, harness/judge per note), not mcp-atlas-v1-500 (official 4-minute v1 harness with Gemini 2.5 Pro judge). Checkpoint: OpenAI GPT-5.2 at reasoning effort xhigh (label 'GPT-5.2 (xhigh)').",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.74,
    "source": "https://huggingface.co/Qwen/Qwen3.5-0.8B/blob/2fc06364715b967f1860aea9cf38778875588b17/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-0.8B model card (Apache-2.0, pinned 2fc0636), '### Language' table, section 'Knowledge & STEM (Thinking)', row 'MMLU-Pro', column 'Qwen3-4B-2507' = 74.0. Same-lab column in Qwen's own table (no cell marked as copied); footnote 'Experimental settings: top_p=0.95, top_k=20, presence_penalty=1.5, and temperature=1.0 were used.' Checkpoint: Qwen/Qwen3-4B-Thinking-2507: in the thinking-mode section the 'Qwen3-4B-2507' column is the Thinking-2507 checkpoint (the non-thinking section's column is Instruct-2507); value equals that model's own card (74.0), i.e. carried over from Qwen's earlier run. Same section/row as the existing Qwen3.5-0.8B thinking-mode row (the separate Non-Thinking Mode MMLU-Pro row is not used).",
    "source_id": "card-qwen--qwen3.5-0.8b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-pro",
    "score": 0.565,
    "source": "https://huggingface.co/Qwen/Qwen3.5-0.8B/blob/2fc06364715b967f1860aea9cf38778875588b17/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-0.8B model card (Apache-2.0, pinned 2fc0636), '### Language' table, section 'Knowledge & STEM (Thinking)', row 'MMLU-Pro', column 'Qwen3-1.7B' = 56.5. Same-lab column in Qwen's own table (no cell marked as copied); footnote 'Experimental settings: top_p=0.95, top_k=20, presence_penalty=1.5, and temperature=1.0 were used.' Checkpoint: Qwen/Qwen3-1.7B (April 2025 hybrid) in thinking mode (thinking-mode section). Same section/row as the existing Qwen3.5-0.8B thinking-mode row (the separate Non-Thinking Mode MMLU-Pro row is not used).",
    "source_id": "card-qwen--qwen3.5-0.8b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-pro",
    "score": 0.665,
    "source": "https://huggingface.co/Qwen/Qwen3.5-0.8B/blob/2fc06364715b967f1860aea9cf38778875588b17/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-0.8B model card (Apache-2.0, pinned 2fc0636), '### Language' table, section 'Knowledge & STEM (Thinking)', row 'MMLU-Pro', column 'Qwen3.5-2B' = 66.5. Same-lab column in Qwen's own table (no cell marked as copied); footnote 'Experimental settings: top_p=0.95, top_k=20, presence_penalty=1.5, and temperature=1.0 were used.' Checkpoint: Qwen/Qwen3.5-2B in thinking mode (thinking-mode section). Same section/row as the existing Qwen3.5-0.8B thinking-mode row (the separate Non-Thinking Mode MMLU-Pro row is not used).",
    "source_id": "card-qwen--qwen3.5-0.8b"
  },
  {
    "model": "ministral-3-14b",
    "bench": "math-subset-unspecified",
    "score": 0.904,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512/blob/5b26027e7b19eeb4b7352e1fed3926375dd2cb4d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 8B Instruct 2512 card (Apache-2.0, pinned 5b26027), '## Benchmark Results' / '### Instruct' table, row 'Ministral 3 14B', column 'MATH Maj@1' = 0.904 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Instruct section rows are the Instruct 2512 checkpoints. Checkpoint: mistralai/Ministral-3-14B-Instruct-2512. Full MATH vs MATH-500 subset not stated (same as the existing Ministral 3 8B row). The same value is printed in that model's own card (one Mistral table reprinted in each Ministral 3 card). Qwen/Gemma rows not used (no statement that Mistral ran them).",
    "source_id": "card-mistralai-ministral-3-8b-instruct-2512"
  },
  {
    "model": "ministral-3-3b",
    "bench": "math-subset-unspecified",
    "score": 0.83,
    "source": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512/blob/5b26027e7b19eeb4b7352e1fed3926375dd2cb4d/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ministral 3 8B Instruct 2512 card (Apache-2.0, pinned 5b26027), '## Benchmark Results' / '### Instruct' table, row 'Ministral 3 3B', column 'MATH Maj@1' = 0.830 (already a fraction). Same-lab row in Mistral's own table ('We compare Ministral 3 to similar sized models.'); the Instruct section rows are the Instruct 2512 checkpoints. Checkpoint: mistralai/Ministral-3-3B-Instruct-2512. Full MATH vs MATH-500 subset not stated (same as the existing Ministral 3 8B row). The same value is printed in that model's own card (one Mistral table reprinted in each Ministral 3 card). Qwen/Gemma rows not used (no statement that Mistral ran them).",
    "source_id": "card-mistralai-ministral-3-8b-instruct-2512"
  },
  {
    "model": "llama-2-7b",
    "bench": "natural-questions",
    "score": 0.247,
    "source": "https://arxiv.org/html/2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B paper (arXiv:2310.06825v1, CC BY 4.0) Table 2, row 'LLaMA 2 7B' (Pretrained), column 'NQ' = 24.7% (NaturalQuestions, 5-shot per Section 3 'World Knowledge (5-shot): NaturalQuestions, TriviaQA'). Sec. 3: 'We compare Mistral 7B to Llama, and re-run all benchmarks with our own evaluation pipeline for fair comparison.' Checkpoint: Base (pretrained) Llama 2 7B, not Llama 2 Chat. Same column as the existing Code-Llama 7B NQ row; the other columns of these rows are already recorded.",
    "source_id": "paper-2310.06825v1"
  },
  {
    "model": "llama-2-13b",
    "bench": "natural-questions",
    "score": 0.29,
    "source": "https://arxiv.org/html/2310.06825v1",
    "by": "independent",
    "date": "2023-10-10",
    "note": "Mistral 7B paper (arXiv:2310.06825v1, CC BY 4.0) Table 2, row 'LLaMA 2 13B' (Pretrained), column 'NQ' = 29.0% (NaturalQuestions, 5-shot per Section 3 'World Knowledge (5-shot): NaturalQuestions, TriviaQA'). Sec. 3: 'We compare Mistral 7B to Llama, and re-run all benchmarks with our own evaluation pipeline for fair comparison.' Checkpoint: Base (pretrained) Llama 2 13B, not Llama 2 Chat. Same column as the existing Code-Llama 7B NQ row; the other columns of these rows are already recorded.",
    "source_id": "paper-2310.06825v1"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "natural-questions",
    "score": 0.288,
    "source": "https://arxiv.org/html/2310.06825v1",
    "by": "lab",
    "date": "2023-10-10",
    "note": "Mistral 7B paper (arXiv:2310.06825v1, CC BY 4.0) Table 2, row 'Mistral 7B' (Pretrained), column 'NQ' = 28.8% (NaturalQuestions, 5-shot per Section 3 'World Knowledge (5-shot): NaturalQuestions, TriviaQA'). Sec. 3: 'We compare Mistral 7B to Llama, and re-run all benchmarks with our own evaluation pipeline for fair comparison.' Checkpoint: Pretrained Mistral 7B (v0.1), not Mistral 7B Instruct. Same column as the existing Code-Llama 7B NQ row; the other columns of these rows are already recorded.",
    "source_id": "paper-2310.06825v1"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau2-bench",
    "score": 0.871,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Language' benchmark table, row 'TAU2-Bench', column 'GPT5.2' = 87.1. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.' Checkpoint: GPT5.2 (reasoning effort not printed). Same row as the own-model value (Qwen3.5-397B-A17B = 86.7).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau2-bench",
    "score": 0.916,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Language' benchmark table, row 'TAU2-Bench', column 'Claude 4.5 Opus' = 91.6. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.' Checkpoint: Claude 4.5 Opus. Same row as the own-model value (Qwen3.5-397B-A17B = 86.7).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau2-bench",
    "score": 0.854,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Language' benchmark table, row 'TAU2-Bench', column 'Gemini-3 Pro' = 85.4. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.' Checkpoint: Gemini-3 Pro (Feb 2026 card; only Gemini 3 Pro checkpoint was gemini-3-pro-preview). Same row as the own-model value (Qwen3.5-397B-A17B = 86.7).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "kimi-k2-5",
    "bench": "tau2-bench",
    "score": 0.77,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Language' benchmark table, row 'TAU2-Bench', column 'K2.5-1T-A32B' = 77.0. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.' Checkpoint: K2.5-1T-A32B = Kimi K2.5 (1T total, 32B active). Same row as the own-model value (Qwen3.5-397B-A17B = 86.7).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "gpt-5-2",
    "bench": "math-vision",
    "score": 0.83,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Vision Language' benchmark table, row 'MathVision', column 'GPT5.2' = 83.0. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* MathVision: our model’s score is evaluated using a fixed prompt, e.g., “Please reason step by step, and put your final answer within \\boxed{}.” For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: GPT5.2 (reasoning effort not printed). Same row as the own-model value (Qwen3.5-397B-A17B = 88.6).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "math-vision",
    "score": 0.743,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Vision Language' benchmark table, row 'MathVision', column 'Claude 4.5 Opus' = 74.3. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* MathVision: our model’s score is evaluated using a fixed prompt, e.g., “Please reason step by step, and put your final answer within \\boxed{}.” For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: Claude 4.5 Opus. Same row as the own-model value (Qwen3.5-397B-A17B = 88.6).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "math-vision",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Vision Language' benchmark table, row 'MathVision', column 'Gemini-3 Pro' = 86.6. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* MathVision: our model’s score is evaluated using a fixed prompt, e.g., “Please reason step by step, and put your final answer within \\boxed{}.” For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: Gemini-3 Pro (Feb 2026 card; only Gemini 3 Pro checkpoint was gemini-3-pro-preview). Same row as the own-model value (Qwen3.5-397B-A17B = 88.6).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "kimi-k2-5",
    "bench": "math-vision",
    "score": 0.842,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3.5-397B-A17B model card @8472618, 'Vision Language' benchmark table, row 'MathVision', column 'K2.5-1T-A32B' = 84.2. Comparator cell evaluated by the Qwen team under the card's setup; evidence footnote: '* MathVision: our model’s score is evaluated using a fixed prompt, e.g., “Please reason step by step, and put your final answer within \\boxed{}.” For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: K2.5-1T-A32B = Kimi K2.5 (1T total, 32B active). Same row as the own-model value (Qwen3.5-397B-A17B = 88.6).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-verified",
    "score": 0.75,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Verified', column 'Qwen3.5-27B' = 75.0. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Verified', column 'Qwen3.5-35BA3B' = 70.0. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35BA3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-multilingual",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Multilingual', column 'Qwen3.5-27B' = 69.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.672,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Multilingual', column 'Qwen3.6-35BA3B' = 67.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.512,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Pro', column 'Qwen3.5-27B' = 51.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'. Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.357,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Pro', column 'Gemma4-31B' = 35.7. Baseline run by the Qwen team: footnote '* SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'. Checkpoint: Gemma4-31B = Gemma 4 31B instruction-tuned (google/gemma-4-31B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.138,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Pro', column 'Gemma4-26BA4B' = 13.8. Baseline run by the Qwen team: footnote '* SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'. Checkpoint: Gemma4-26BA4B = Gemma 4 26B-A4B instruction-tuned (google/gemma-4-26B-A4B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.495,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SWE-bench Pro', column 'Qwen3.6-35BA3B' = 49.5. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* SWE-Bench Series: Internal agent scaffold (bash + file-edit tools); temp=1.0, top_p=0.95, 200K context window. We correct some problematic tasks in the public set of SWE-bench Pro and evaluate all baselines on the refined benchmark.'. Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "claw-eval-avg",
    "score": 0.643,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'Claw-Eval Avg', column 'Qwen3.5-27B' = 64.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "claw-eval-avg",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'Claw-Eval Avg', column 'Qwen3.6-35BA3B' = 68.7. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "claw-eval-pass3",
    "score": 0.462,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'Claw-Eval Pass^3', column 'Qwen3.5-27B' = 46.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "claw-eval-pass3",
    "score": 0.5,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'Claw-Eval Pass^3', column 'Qwen3.6-35BA3B' = 50.0. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "skillsbench-qwen-78",
    "score": 0.272,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SkillsBench Avg5', column 'Qwen3.5-27B' = 27.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* SkillsBench: Evaluated via OpenCode on 78 tasks (self-contained subset, excluding API-dependent tasks); avg of 5 runs.' (a Qwen-defined subset, so every cell is Qwen's own run). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "skillsbench-qwen-78",
    "score": 0.236,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SkillsBench Avg5', column 'Gemma4-31B' = 23.6. Baseline run by the Qwen team: footnote '* SkillsBench: Evaluated via OpenCode on 78 tasks (self-contained subset, excluding API-dependent tasks); avg of 5 runs.' (a Qwen-defined subset, so every cell is Qwen's own run). Checkpoint: Gemma4-31B = Gemma 4 31B instruction-tuned (google/gemma-4-31B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "skillsbench-qwen-78",
    "score": 0.123,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SkillsBench Avg5', column 'Gemma4-26BA4B' = 12.3. Baseline run by the Qwen team: footnote '* SkillsBench: Evaluated via OpenCode on 78 tasks (self-contained subset, excluding API-dependent tasks); avg of 5 runs.' (a Qwen-defined subset, so every cell is Qwen's own run). Checkpoint: Gemma4-26BA4B = Gemma 4 26B-A4B instruction-tuned (google/gemma-4-26B-A4B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "skillsbench-qwen-78",
    "score": 0.287,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'SkillsBench Avg5', column 'Qwen3.6-35BA3B' = 28.7. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* SkillsBench: Evaluated via OpenCode on 78 tasks (self-contained subset, excluding API-dependent tasks); avg of 5 runs.' (a Qwen-defined subset, so every cell is Qwen's own run). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "qwenclawbench",
    "score": 0.522,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'QwenClawBench', column 'Qwen3.5-27B' = 52.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* QwenClawBench: An internal real-user-distribution Claw agent benchmark (open-sourcing soon); temp=0.6, 256K ctx.' (internal, unreleased benchmark, so every cell is Qwen's own run). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "qwenclawbench",
    "score": 0.417,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'QwenClawBench', column 'Gemma4-31B' = 41.7. Baseline run by the Qwen team: footnote '* QwenClawBench: An internal real-user-distribution Claw agent benchmark (open-sourcing soon); temp=0.6, 256K ctx.' (internal, unreleased benchmark, so every cell is Qwen's own run). Checkpoint: Gemma4-31B = Gemma 4 31B instruction-tuned (google/gemma-4-31B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "qwenclawbench",
    "score": 0.387,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'QwenClawBench', column 'Gemma4-26BA4B' = 38.7. Baseline run by the Qwen team: footnote '* QwenClawBench: An internal real-user-distribution Claw agent benchmark (open-sourcing soon); temp=0.6, 256K ctx.' (internal, unreleased benchmark, so every cell is Qwen's own run). Checkpoint: Gemma4-26BA4B = Gemma 4 26B-A4B instruction-tuned (google/gemma-4-26B-A4B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "qwenclawbench",
    "score": 0.526,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'QwenClawBench', column 'Qwen3.6-35BA3B' = 52.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* QwenClawBench: An internal real-user-distribution Claw agent benchmark (open-sourcing soon); temp=0.6, 256K ctx.' (internal, unreleased benchmark, so every cell is Qwen's own run). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "nl2repo",
    "score": 0.273,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'NL2Repo', column 'Qwen3.5-27B' = 27.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).'. Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-31b",
    "bench": "nl2repo",
    "score": 0.155,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'NL2Repo', column 'Gemma4-31B' = 15.5. Baseline run by the Qwen team: footnote '* NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).'. Checkpoint: Gemma4-31B = Gemma 4 31B instruction-tuned (google/gemma-4-31B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "nl2repo",
    "score": 0.116,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "independent",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'NL2Repo', column 'Gemma4-26BA4B' = 11.6. Baseline run by the Qwen team: footnote '* NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).'. Checkpoint: Gemma4-26BA4B = Gemma 4 26B-A4B instruction-tuned (google/gemma-4-26B-A4B-it), the only Gemma 4 release of that size.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "nl2repo",
    "score": 0.294,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'NL2Repo', column 'Qwen3.6-35BA3B' = 29.4. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). footnote '* NL2Repo: Others are evaluated via Claude Code (temp=1.0, top_p=0.95, max_turns=900).'. Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "deepplanning",
    "score": 0.226,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'DeepPlanning', column 'Qwen3.5-27B' = 22.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "deepplanning",
    "score": 0.228,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'DeepPlanning', column 'Qwen3.5-35BA3B' = 22.8. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35BA3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "mmlu-pro",
    "score": 0.861,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'MMLU-Pro', column 'Qwen3.5-27B' = 86.1. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'MMLU-Pro', column 'Qwen3.5-35BA3B' = 85.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35BA3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hle",
    "score": 0.243,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'HLE', column 'Qwen3.5-27B' = 24.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hle",
    "score": 0.224,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'HLE', column 'Qwen3.5-35BA3B' = 22.4. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35BA3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "livecodebench-v6",
    "score": 0.807,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'LiveCodeBench v6', column 'Qwen3.5-27B' = 80.7. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'LiveCodeBench v6', column 'Qwen3.5-35BA3B' = 74.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35BA3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hmmt-feb-2026",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'HMMT Feb 26', column 'Qwen3.5-27B' = 84.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "hmmt-feb-2026",
    "score": 0.836,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'HMMT Feb 26', column 'Qwen3.6-35BA3B' = 83.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "imo-answerbench",
    "score": 0.799,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'IMOAnswerBench', column 'Qwen3.5-27B' = 79.9. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "imo-answerbench",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'IMOAnswerBench', column 'Qwen3.6-35BA3B' = 78.9. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "aime-2026",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'AIME26', column 'Qwen3.5-27B' = 92.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "aime-2026",
    "score": 0.927,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, language benchmark table, row 'AIME26', column 'Qwen3.6-35BA3B' = 92.7. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.6-35BA3B = Qwen/Qwen3.6-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "zerobench-sub",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'ZEROBench_sub', column 'Qwen3.5-27B' = 36.2. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "zerobench-sub",
    "score": 0.341,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'ZEROBench_sub', column 'Qwen3.5-35B-A3B' = 34.1. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "hallusionbench",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'HallusionBench', column 'Qwen3.5-27B' = 70.0. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hallusionbench",
    "score": 0.679,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'HallusionBench', column 'Qwen3.5-35B-A3B' = 67.9. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.889,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'OmniDocBench1.5', column 'Qwen3.5-27B' = 88.9. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "omnidocbench-1-5-overall",
    "score": 0.893,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'OmniDocBench1.5', column 'Qwen3.5-35B-A3B' = 89.3. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "ai2d-test",
    "score": 0.929,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'AI2D_TEST', column 'Qwen3.5-27B' = 92.9. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "ai2d-test",
    "score": 0.926,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'AI2D_TEST', column 'Qwen3.5-35B-A3B' = 92.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "videomme-without-sub",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'VideoMME(w/o sub.)', column 'Qwen3.5-27B' = 82.8. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "videomme-without-sub",
    "score": 0.825,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'VideoMME(w/o sub.)', column 'Qwen3.5-35B-A3B' = 82.5. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "lvbench",
    "score": 0.736,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'LVBench', column 'Qwen3.5-27B' = 73.6. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-27B = Qwen/Qwen3.5-27B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "lvbench",
    "score": 0.714,
    "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md",
    "by": "lab",
    "date": "2026-04-16",
    "note": "Qwen3.6-35B-A3B model card @995ad96, vision-language benchmark table, row 'LVBench', column 'Qwen3.5-35B-A3B' = 71.4. Same-family Qwen column in Qwen's own table (Qwen3.5 columns are Qwen's re-evaluations under the Qwen3.6 setup; thinking mode default). Checkpoint: Qwen3.5-35B-A3B = Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.6-35b-a3b"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "mbpp-plus",
    "score": 0.72,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R7B', column 'MBPP+' = 72.0. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "mbpp-plus",
    "score": 0.743,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R Refresh', column 'MBPP+' = 74.3. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "mbpp-plus",
    "score": 0.788,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R+ Refresh', column 'MBPP+' = 78.8. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "mbpp-plus",
    "score": 0.81,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.3 70B Instruct', column 'MBPP+' = 86.0 / 81.0 (internal reproduction = second value 81.0). Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "mbpp-plus",
    "score": 0.886,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Qwen 2.5 72B Instruct', column 'MBPP+' = 88.6. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "mbpp-plus",
    "score": 0.87,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.1 405B Instruct', column 'MBPP+' = 88.6 / 87.0 (internal reproduction = second value 87.0). Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "mbpp-plus",
    "score": 0.9,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'DeepSeek V3', column 'MBPP+' = 90.0. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.09,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R7B', column 'LiveCodeBench' = 9.0. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.11,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R Refresh', column 'LiveCodeBench' = 11.0. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.144,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R+ Refresh', column 'LiveCodeBench' = 14.4. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.329,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.3 70B Instruct', column 'LiveCodeBench' = 32.9. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.263,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Qwen 2.5 72B Instruct', column 'LiveCodeBench' = 26.3. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.293,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.1 405B Instruct', column 'LiveCodeBench' = 29.3. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "livecodebench-v5-202410-202502",
    "score": 0.335,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'DeepSeek V3', column 'LiveCodeBench' = 33.5. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "bigcodebench-instruct",
    "score": 0.309,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R7B', column 'BigCodeBench' = 30.9. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "bigcodebench-instruct",
    "score": 0.343,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R Refresh', column 'BigCodeBench' = 34.3. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "bigcodebench-instruct",
    "score": 0.258,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R+ Refresh', column 'BigCodeBench' = 25.8. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bigcodebench-instruct",
    "score": 0.419,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.3 70B Instruct', column 'BigCodeBench' = 46.9 / 41.9 (internal reproduction = second value 41.9). Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "bigcodebench-instruct",
    "score": 0.436,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Qwen 2.5 72B Instruct', column 'BigCodeBench' = 45.8 / 43.6 (internal reproduction = second value 43.6). Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "bigcodebench-instruct",
    "score": 0.462,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.1 405B Instruct', column 'BigCodeBench' = 46.2. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "bigcodebench-instruct",
    "score": 0.486,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'DeepSeek V3', column 'BigCodeBench' = 50.0 / 48.6 (internal reproduction = second value 48.6). Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "repoqa",
    "score": 0.696,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R7B', column 'RepoQA' = 69.6. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "repoqa",
    "score": 0.732,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R Refresh', column 'RepoQA' = 73.2. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "repoqa",
    "score": 0.77,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Command R+ Refresh', column 'RepoQA' = 77.0. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "repoqa",
    "score": 0.856,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.3 70B Instruct', column 'RepoQA' = 85.6. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "repoqa",
    "score": 0.832,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Qwen 2.5 72B Instruct', column 'RepoQA' = 83.2. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "repoqa",
    "score": 0.904,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'Llama 3.1 405B Instruct', column 'RepoQA' = 90.4. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "repoqa",
    "score": 0.922,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 12, row 'DeepSeek V3', column 'RepoQA' = 92.2. Evidence the cell is Cohere's own run: Table 12 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates external value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "swe-bench-verified",
    "score": 0.036,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R7B', column 'SWE-Bench Verified' = 3.6. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "swe-bench-verified",
    "score": 0.116,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R Refresh', column 'SWE-Bench Verified' = 11.6. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "swe-bench-verified",
    "score": 0.17,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R+ Refresh', column 'SWE-Bench Verified' = 17.0. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "swe-bench-verified",
    "score": 0.294,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Llama 3.3 70B Instruct', column 'SWE-Bench Verified' = 29.4. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "swe-bench-verified",
    "score": 0.33,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Qwen 2.5 72B Instruct', column 'SWE-Bench Verified' = 33.0. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "swe-bench-verified",
    "score": 0.334,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Llama 3.1 405B Instruct', column 'SWE-Bench Verified' = 33.4. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "swe-bench-verified",
    "score": 0.458,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'DeepSeek V3', column 'SWE-Bench Verified' = 42.0 / 45.8 (internal reproduction = second value 45.8). Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "aider-polyglot",
    "score": 0.027,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R7B', column 'Aider Polyglot' = 2.7. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "aider-polyglot",
    "score": 0.018,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R Refresh', column 'Aider Polyglot' = 1.8. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "aider-polyglot",
    "score": 0.022,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Command R+ Refresh', column 'Aider Polyglot' = 2.2. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "aider-polyglot",
    "score": 0.084,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Llama 3.3 70B Instruct', column 'Aider Polyglot' = 8.4. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "aider-polyglot",
    "score": 0.08,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Qwen 2.5 72B Instruct', column 'Aider Polyglot' = 8.0. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "aider-polyglot",
    "score": 0.138,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'Llama 3.1 405B Instruct', column 'Aider Polyglot' = 13.8. Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "aider-polyglot",
    "score": 0.516,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 13, row 'DeepSeek V3', column 'Aider Polyglot' = 49.6 / 51.6 (internal reproduction = second value 51.6). Evidence the cell is Cohere's own run: Table 13 caption: 'All results are internal reproductions using an identical prompt except where '/' indicates externally reported value first and internal reproduction second.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "bird-sql-dev",
    "score": 0.422,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Command R7B', column 'Bird Dev' = 42.2. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-08-2024",
    "bench": "bird-sql-dev",
    "score": 0.473,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Command R Refresh', column 'Bird Dev' = 47.3. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: Cohere's own previous model; 'Refresh' = the 08-2024 update of Command R (command-r-08-2024), the only refresh before Command A.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "bird-sql-dev",
    "score": 0.527,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Command R+ Refresh', column 'Bird Dev' = 52.7. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "bird-sql-dev",
    "score": 0.58,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Llama 3.3 70B Instruct', column 'Bird Dev' = 58.0. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "bird-sql-dev",
    "score": 0.501,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Qwen 2.5 72B Instruct', column 'Bird Dev' = 50.1. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: Qwen/Qwen2.5-72B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "bird-sql-dev",
    "score": 0.594,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'Llama 3.1 405B Instruct', column 'Bird Dev' = 59.4. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "bird-sql-dev",
    "score": 0.531,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 14, row 'DeepSeek V3', column 'Bird Dev' = 53.1. Evidence the cell is Cohere's own run: Table 14 caption: 'All results are internal reproductions using an identical prompt.'. Checkpoint: original DeepSeek-V3 (Dec 2024): its externally reported values in Table 13 (SWE-bench Verified 42.0, Aider Polyglot 49.6) equal DeepSeek's own Dec-2024 V3 figures already catalogued for deepseek-v3.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "aime-2024",
    "score": 0.2,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 15, row 'Llama 3.3 70B Instruct', column 'AIME (2024)' = 20.0*. Evidence the cell is Cohere's own run: Table 15 caption: 'Results for external models are taken from officially-reported sources, unless indicated with an asterisk (*), which denotes internal evaluation since official public results were not available.' This cell is printed '20.0*' (row 'Llama 3.3 70B'). Checkpoint: meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "ruler-128k",
    "score": 0.646,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 21, row 'Command R+ Refresh', column '128k' = 64.6. Evidence the cell is Cohere's own run: same-family Cohere model in Cohere's own long-context table (no copied-value marker). Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "longbench-v2",
    "score": 0.278,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 22, row 'Command R+ Refresh', column 'Overall' = 27.8. Evidence the cell is Cohere's own run: same-family Cohere model in Cohere's own long-context table (no copied-value marker). Checkpoint: Cohere's own previous model ('our previous Command R+ Refresh model'); 'Refresh' = the 08-2024 update (command-r-plus-08-2024).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "simple-evals-mmlu",
    "score": 0.652,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 3, row 'Command R7B', column 'MMLU' = 65.2. Evidence the cell is Cohere's own run: Table 3 (standard academic benchmarks; §4.1: 'Where applicable (MMLU, MMLU-Pro, GPQA), we follow the simple-evals implementation, including data, task settings, prompting, and answer parsing'; IFEval is the mean of prompt- and instruction-level strict accuracy per the caption). Same-family Cohere model, no copied-value marker. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "mmlu-pro",
    "score": 0.424,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 3, row 'Command R7B', column 'MMLU-Pro' = 42.4. Evidence the cell is Cohere's own run: Table 3 (standard academic benchmarks; §4.1: 'Where applicable (MMLU, MMLU-Pro, GPQA), we follow the simple-evals implementation, including data, task settings, prompting, and answer parsing'; IFEval is the mean of prompt- and instruction-level strict accuracy per the caption). Same-family Cohere model, no copied-value marker. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "simple-evals-gpqa-diamond",
    "score": 0.263,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 3, row 'Command R7B', column 'GPQA' = 26.3. Evidence the cell is Cohere's own run: Table 3 (standard academic benchmarks; §4.1: 'Where applicable (MMLU, MMLU-Pro, GPQA), we follow the simple-evals implementation, including data, task settings, prompting, and answer parsing'; IFEval is the mean of prompt- and instruction-level strict accuracy per the caption). Same-family Cohere model, no copied-value marker. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "ifeval-strict-mean",
    "score": 0.779,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 3, row 'Command R7B', column 'IFEval' = 77.9. Evidence the cell is Cohere's own run: Table 3 (standard academic benchmarks; §4.1: 'Where applicable (MMLU, MMLU-Pro, GPQA), we follow the simple-evals implementation, including data, task settings, prompting, and answer parsing'; IFEval is the mean of prompt- and instruction-level strict accuracy per the caption). Same-family Cohere model, no copied-value marker. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "command-r7b-12-2024",
    "bench": "infobench",
    "score": 0.856,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "lab",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 3, row 'Command R7B', column 'InFoBench' = 85.6. Evidence the cell is Cohere's own run: Table 3 (standard academic benchmarks; §4.1: 'Where applicable (MMLU, MMLU-Pro, GPQA), we follow the simple-evals implementation, including data, task settings, prompting, and answer parsing'; IFEval is the mean of prompt- and instruction-level strict accuracy per the caption). Same-family Cohere model, no copied-value marker. Checkpoint: Cohere's own model (report: 'This technical report describes the development of Command A and Command R7B'); the only Command R7B release, c4ai-command-r7b-12-2024.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "tau2-bench",
    "score": 0.852,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'τ²-Bench', column 'DeepSeek V3.2', printed '80.3 (85.2*)' -> starred value 85.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: deepseek-ai/DeepSeek-V3.2 (37B active / 671B total in the card's parameter rows); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "tau2-bench",
    "score": 0.743,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'τ²-Bench', column 'Kimi K2 Thinking / K2.5 (first value)', printed '74.3*/85.4*' -> starred value 74.3 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-5",
    "bench": "tau2-bench",
    "score": 0.854,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'τ²-Bench', column 'Kimi K2 Thinking / K2.5 (second value)', printed '74.3*/85.4*' -> starred value 85.4 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2.5, moonshotai/Kimi-K2.5 (second value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "tau2-bench",
    "score": 0.866,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'τ²-Bench', column 'MiniMax M2.1', printed '86.6*' -> starred value 86.6 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "tau2-bench",
    "score": 0.841,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'τ²-Bench', column 'MiMo-V2 Flash', printed '80.3 (84.1*)' -> starred value 84.1 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "browsecomp",
    "score": 0.415,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'BrowseComp', column 'Kimi K2 Thinking / K2.5 (first value)', printed '41.5* / 60.6' -> starred value 41.5 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-5",
    "bench": "browsecomp-zh",
    "score": 0.623,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'BrowseComp-ZH', column 'Kimi K2 Thinking / K2.5 (second value)', printed '62.3 / 62.3*' -> starred value 62.3 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2.5, moonshotai/Kimi-K2.5 (second value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "browsecomp-zh",
    "score": 0.478,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'BrowseComp-ZH', column 'MiniMax M2.1', printed '47.8*' -> starred value 47.8 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "browsecomp-zh",
    "score": 0.512,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'BrowseComp-ZH', column 'MiMo-V2 Flash', printed '51.2*' -> starred value 51.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "gaia-text-only",
    "score": 0.751,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'DeepSeek V3.2', printed '75.1*' -> starred value 75.1 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: deepseek-ai/DeepSeek-V3.2 (37B active / 671B total in the card's parameter rows); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "gaia-text-only",
    "score": 0.756,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'Kimi K2 Thinking / K2.5 (first value)', printed '75.6*/75.9*' -> starred value 75.6 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-5",
    "bench": "gaia-text-only",
    "score": 0.759,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'Kimi K2 Thinking / K2.5 (second value)', printed '75.6*/75.9*' -> starred value 75.9 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2.5, moonshotai/Kimi-K2.5 (second value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "glm-4-7",
    "bench": "gaia-text-only",
    "score": 0.619,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'GLM-4.7', printed '61.9*' -> starred value 61.9 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: zai-org/GLM-4.7 (32B active / 355B total); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "gaia-text-only",
    "score": 0.643,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'MiniMax M2.1', printed '64.3*' -> starred value 64.3 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "gaia-text-only",
    "score": 0.782,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'GAIA (no file)', column 'MiMo-V2 Flash', printed '78.2*' -> starred value 78.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "researchrubrics",
    "score": 0.558,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'DeepSeek V3.2', printed '55.8*' -> starred value 55.8 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: deepseek-ai/DeepSeek-V3.2 (37B active / 671B total in the card's parameter rows); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "researchrubrics",
    "score": 0.562,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'Kimi K2 Thinking / K2.5 (first value)', printed '56.2*/59.5*' -> starred value 56.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-5",
    "bench": "researchrubrics",
    "score": 0.595,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'Kimi K2 Thinking / K2.5 (second value)', printed '56.2*/59.5*' -> starred value 59.5 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2.5, moonshotai/Kimi-K2.5 (second value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "glm-4-7",
    "bench": "researchrubrics",
    "score": 0.62,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'GLM-4.7', printed '62.0*' -> starred value 62.0 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: zai-org/GLM-4.7 (32B active / 355B total); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "researchrubrics",
    "score": 0.602,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'MiniMax M2.1', printed '60.2*' -> starred value 60.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "researchrubrics",
    "score": 0.543,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'ResearchRubrics', column 'MiMo-V2 Flash', printed '54.3*' -> starred value 54.3 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "aime-2025",
    "score": 0.951,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'AIME 2025', column 'MiMo-V2 Flash', printed '94.1 (95.1*)' -> starred value 95.1 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "hmmt-feb-2025",
    "score": 0.71,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'HMMT 2025 (Feb.)', column 'MiniMax M2.1', printed '71.0*' -> starred value 71.0 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "hmmt-feb-2025",
    "score": 0.954,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'HMMT 2025 (Feb.)', column 'MiMo-V2 Flash', printed '84.4 (95.4*)' -> starred value 95.4 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "hmmt-nov-2025",
    "score": 0.892,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'HMMT 2025 (Nov.)', column 'Kimi K2 Thinking / K2.5 (first value)', printed '89.2*/—' -> starred value 89.2 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "hmmt-nov-2025",
    "score": 0.743,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'HMMT 2025 (Nov.)', column 'MiniMax M2.1', printed '74.3*' -> starred value 74.3 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "hmmt-nov-2025",
    "score": 0.91,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'HMMT 2025 (Nov.)', column 'MiMo-V2 Flash', printed '91.0*' -> starred value 91.0 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "minimax-m2-1",
    "bench": "imo-answerbench",
    "score": 0.604,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'IMOAnswerBench', column 'MiniMax M2.1', printed '60.4*' -> starred value 60.4 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: MiniMaxAI/MiniMax-M2.1 (10B active / 230B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "imo-answerbench",
    "score": 0.809,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'IMOAnswerBench', column 'MiMo-V2 Flash', printed '80.9*' -> starred value 80.9 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.816,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'LiveCodeBench-V6', column 'MiMo-V2 Flash', printed '80.6 (81.6*)' -> starred value 81.6 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: XiaomiMiMo/MiMo-V2-Flash (15B active / 309B total) - proposed comparison-only model; reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "kimi-k2-thinking",
    "bench": "terminal-bench-2",
    "score": 0.357,
    "source": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/ab446a3de5e171ea341227e24bb1f090e1b771f7/README.md",
    "by": "independent",
    "date": "2026-03-17",
    "note": "Step 3.5 Flash model card @ab446a3, 'Detailed Benchmarks' table, row 'Terminal-Bench 2.0', column 'Kimi K2 Thinking / K2.5 (first value)', printed '35.7*/50.8' -> starred value 35.7 (StepFun's reproduction under the Step 3.5 Flash test conditions). Card note 2: '\"*\" indicates the original score was inaccessible or lower than our reproduced, so we report the evaluation under the same test conditions as Step 3.5 Flash to ensure fair comparability.' Only starred cells are used; an unstarred value in the same cell is the competitor's own published figure and is not used. Checkpoint: Kimi K2 Thinking, moonshotai/Kimi-K2-Thinking (column header 'Kimi K2 Thinking / K2.5'; first value of each 'a/b' cell); reasoning setting not printed.",
    "source_id": "card2-stepfun-ai-step-3-5-flash"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "wildbench-v2",
    "score": 0.556,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'Instruction Following / Chat / Tone' table, row 'Small 3.1 24B Instruct', column 'Wildbench v2' = 55.6%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "arena-hard-v2",
    "score": 0.1956,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'Instruction Following / Chat / Tone' table, row 'Small 3.1 24B Instruct', column 'Arena Hard v2' = 19.56%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mmlu",
    "score": 0.8062,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'MMLU' = 80.62%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mmlu-pro",
    "score": 0.6676,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'MMLU Pro (5-shot CoT)' = 66.76%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "math-subset-unspecified",
    "score": 0.693,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'MATH' = 69.30%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "gpqa-main",
    "score": 0.4442,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'GPQA Main (5-shot CoT)' = 44.42%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "gpqa-diamond",
    "score": 0.4596,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'GPQA Diamond (5-shot CoT )' = 45.96%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mbpp-plus-pass5",
    "score": 0.7463,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'MBPP Plus - Pass@5' = 74.63%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "humaneval-plus-pass5",
    "score": 0.8899,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'HumanEval Plus - Pass@5' = 88.99%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "simpleqa",
    "score": 0.1043,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'STEM' table, row 'Small 3.1 24B Instruct', column 'SimpleQA (TotalAcc)' = 10.43%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "mmmu",
    "score": 0.64,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'Vision' table, row 'Small 3.1 24B Instruct', column 'MMMU' = 64.00%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "chartqa",
    "score": 0.8624,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'Vision' table, row 'Small 3.1 24B Instruct', column 'ChartQA' = 86.24%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "docvqa",
    "score": 0.9408,
    "source": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506/blob/95a6d26c4bfb886c58daf9d3f7332c857cb27b43/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "Mistral-Small-3.2-24B-Instruct-2506 model card @95a6d26, 'Vision' table, row 'Small 3.1 24B Instruct', column 'DocVQA' = 94.08%. Same-lab previous version in Mistral's own comparison table ('We compare Mistral-Small-3.2-24B to [Mistral-Small-3.1-24B-Instruct-2503]'), same row/settings as the Small 3.2 cell. Checkpoint: mistralai/Mistral-Small-3.1-24B-Instruct-2503 (linked in the card).",
    "source_id": "card2-mistralai-mistral-small-3-2-24b-instruct-2506"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmlu",
    "score": 0.6484,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU', column 'LFM2-8B-A1B' = 64.84. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu",
    "score": 0.6035,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU', column 'Llama-3.2-3B' = 60.35. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmlu",
    "score": 0.5835,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU', column 'Gemma-3-4B' = 58.35. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu",
    "score": 0.7225,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU', column 'Qwen3-4B' = 72.25. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmlu-pro",
    "score": 0.3742,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU-Pro', column 'LFM2-8B-A1B' = 37.42. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmlu-pro",
    "score": 0.2225,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU-Pro', column 'Llama-3.2-3B' = 22.25. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmlu-pro",
    "score": 0.3476,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU-Pro', column 'Gemma-3-4B' = 34.76. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-pro",
    "score": 0.5231,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU-Pro', column 'Qwen3-4B' = 52.31. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "gpqa-diamond",
    "score": 0.2929,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GPQA', column 'LFM2-8B-A1B' = 29.29. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gpqa-diamond",
    "score": 0.306,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GPQA', column 'Llama-3.2-3B' = 30.60. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "gpqa-diamond",
    "score": 0.2951,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GPQA', column 'Gemma-3-4B' = 29.51. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "gpqa-diamond",
    "score": 0.3485,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GPQA', column 'Qwen3-4B' = 34.85. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "ifeval-mean-four",
    "score": 0.7758,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFEval', column 'LFM2-8B-A1B' = 77.58. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "ifeval-mean-four",
    "score": 0.7143,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFEval', column 'Llama-3.2-3B' = 71.43. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "ifeval-mean-four",
    "score": 0.7685,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFEval', column 'Gemma-3-4B' = 76.85. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "ifeval-mean-four",
    "score": 0.8562,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFEval', column 'Qwen3-4B' = 85.62. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "ifbench-unspecified",
    "score": 0.2585,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFBench', column 'LFM2-8B-A1B' = 25.85. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "ifbench-unspecified",
    "score": 0.2078,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFBench', column 'Llama-3.2-3B' = 20.78. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "ifbench-unspecified",
    "score": 0.2353,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFBench', column 'Gemma-3-4B' = 23.53. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "ifbench-unspecified",
    "score": 0.3028,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFBench', column 'Qwen3-4B' = 30.28. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "multi-if-three-turn-average",
    "score": 0.5819,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'Multi-IF', column 'LFM2-8B-A1B' = 58.19. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "multi-if-three-turn-average",
    "score": 0.5091,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'Multi-IF', column 'Llama-3.2-3B' = 50.91. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "multi-if-three-turn-average",
    "score": 0.6661,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'Multi-IF', column 'Gemma-3-4B' = 66.61. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "multi-if-three-turn-average",
    "score": 0.7554,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'Multi-IF', column 'Qwen3-4B' = 75.54. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "gsm8k-loose-substring",
    "score": 0.8438,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSM8K', column 'LFM2-8B-A1B' = 84.38. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gsm8k-loose-substring",
    "score": 0.7521,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSM8K', column 'Llama-3.2-3B' = 75.21. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "gsm8k-loose-substring",
    "score": 0.8992,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSM8K', column 'Gemma-3-4B' = 89.92. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "gsm8k-loose-substring",
    "score": 0.6846,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSM8K', column 'Qwen3-4B' = 68.46. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "gsm-plus",
    "score": 0.6476,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSMPlus', column 'LFM2-8B-A1B' = 64.76. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gsm-plus",
    "score": 0.3868,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSMPlus', column 'Llama-3.2-3B' = 38.68. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "gsm-plus",
    "score": 0.6838,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSMPlus', column 'Gemma-3-4B' = 68.38. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "gsm-plus",
    "score": 0.5616,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSMPlus', column 'Qwen3-4B' = 56.16. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "math-500",
    "score": 0.742,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MATH 500', column 'LFM2-8B-A1B' = 74.20. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "math-500",
    "score": 0.412,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MATH 500', column 'Llama-3.2-3B' = 41.20. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "math-500",
    "score": 0.732,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MATH 500', column 'Gemma-3-4B' = 73.20. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "math-500",
    "score": 0.856,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MATH 500', column 'Qwen3-4B' = 85.60. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.5526,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMMLU', column 'LFM2-8B-A1B' = 55.26. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.4792,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMMLU', column 'Llama-3.2-3B' = 47.92. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.5014,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMMLU', column 'Gemma-3-4B' = 50.14. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.6067,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMMLU', column 'Qwen3-4B' = 60.67. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-8b-a1b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.724,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MGSM', column 'LFM2-8B-A1B' = 72.40. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: LiquidAI/LFM2-8B-A1B (Liquid AI own model; catalogued comparison-only entry).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.6168,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MGSM', column 'Llama-3.2-3B' = 61.68. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: meta-llama/Llama-3.2-3B-Instruct (Appendix C: chat template applied, so the instruct checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "gemma-3-4b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.8728,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MGSM', column 'Gemma-3-4B' = 87.28. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: google/gemma-3-4b-it (chat template applied, so the instruction-tuned checkpoint).",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "qwen3-4b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.8176,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MGSM', column 'Qwen3-4B' = 81.76. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row and harness as the existing SmolLM3-3B cell (Appendix C implementation). Checkpoint: original Qwen/Qwen3-4B (the report names the 2507 checkpoint explicitly elsewhere, e.g. 'Qwen3-4B-Instruct-2507' in its SFT ensemble; here only 'Qwen3-4B'); thinking toggle not specified.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "bfcl-version-unspecified",
    "score": 0.661,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 7, row 'BFCL (0-shot)', column 'Phi-3.5-Mini 3.8b' = 66.1. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.506,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 7, row 'IFEval (0-shot)', column 'Phi-3.5-Mini 3.8b' = 50.6. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "humaneval",
    "score": 0.701,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'HumanEval (0-Shot)', column 'Phi-3.5-Mini 3.8b' = 70.1. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mbpp",
    "score": 0.7,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'MBPP (3-Shot)', column 'Phi-3.5-Mini 3.8b' = 70.0. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.404,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'BigCodeBench Completion (0-Shot)', column 'Phi-3.5-Mini 3.8b' = 40.4. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.143,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'BigCodeBench instruct (0-Shot)', column 'Phi-3.5-Mini 3.8b' = 14.3. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "humaneval-plus",
    "score": 0.628,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'HumanEval+ (0-Shot)', column 'Phi-3.5-Mini 3.8b' = 62.8. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "mbpp-plus",
    "score": 0.638,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 8, row 'MBPP+ (3-Shot)', column 'Phi-3.5-Mini 3.8b' = 63.8. Same-lab previous model ('Compared to the predecessor Phi-3.5-Mini ...' in §4.2.1) in Microsoft's own table; column header 'Phi-3.5-Mini 3.8b'. Checkpoint: microsoft/Phi-3.5-mini-instruct. Same row and shots as the Phi-4-Mini cell.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2024",
    "score": 0.533,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Qwen-7B', column 'AIME' = 53.3. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B. AIME = AIME 2024 per §4.2.3 ('We show results on AIME 2024 [MAA24], MATH-500 ..., and GPQA Diamond').",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-500",
    "score": 0.914,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Qwen-7B', column 'MATH-500' = 91.4. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "gpqa-diamond",
    "score": 0.495,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Qwen-7B', column 'GPQA Diamond' = 49.5. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "aime-2024",
    "score": 0.433,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Llama-8B', column 'AIME' = 43.3. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B. AIME = AIME 2024 per §4.2.3 ('We show results on AIME 2024 [MAA24], MATH-500 ..., and GPQA Diamond').",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "math-500",
    "score": 0.869,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Llama-8B', column 'MATH-500' = 86.9. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "gpqa-diamond",
    "score": 0.473,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'DeepSeek-R1-Distill-Llama-8B', column 'GPQA Diamond' = 47.3. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Llama-8B.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "aime-2024",
    "score": 0.067,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'Llama-3.2-3B-Instruct', column 'AIME' = 6.7. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: meta-llama/Llama-3.2-3B-Instruct. AIME = AIME 2024 per §4.2.3 ('We show results on AIME 2024 [MAA24], MATH-500 ..., and GPQA Diamond').",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "math-500",
    "score": 0.444,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'Llama-3.2-3B-Instruct', column 'MATH-500' = 44.4. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: meta-llama/Llama-3.2-3B-Instruct.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "llama-3-2-3b",
    "bench": "gpqa-diamond",
    "score": 0.253,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "independent",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'Llama-3.2-3B-Instruct', column 'GPQA Diamond' = 25.3. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: meta-llama/Llama-3.2-3B-Instruct.",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "phi-4-mini",
    "bench": "aime-2024",
    "score": 0.1,
    "source": "https://arxiv.org/html/2503.01743v2",
    "by": "lab",
    "date": "2025-03-07",
    "note": "Phi-4-Mini technical report (Microsoft, arXiv 2503.01743v2, CC BY 4.0), Table 9 (CoT reasoning), row 'Phi-4-Mini', column 'AIME' = 10.0. Table 9 caption: 'An asterisk (*) indicates results taken directly from the published reports, while the remaining results were reproduced in our work.' This cell has no asterisk. Checkpoint: microsoft/Phi-4-mini-instruct (the plain Phi-4-Mini row, not the reasoning-enhanced rows). AIME = AIME 2024 per §4.2.3 ('We show results on AIME 2024 [MAA24], MATH-500 ..., and GPQA Diamond').",
    "source_id": "report-2503.01743v2"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-pro",
    "score": 0.752,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'MMLU-Pro', column 'Qwen3-235B-A22B Non-thinking' = 75.2. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "simpleqa",
    "score": 0.122,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'SimpleQA', column 'Qwen3-235B-A22B Non-thinking' = 12.2. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "chinese-simpleqa",
    "score": 0.608,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'CSimpleQA', column 'Qwen3-235B-A22B Non-thinking' = 60.8. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aime-2025",
    "score": 0.247,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'AIME25', column 'Qwen3-235B-A22B Non-thinking' = 24.7. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "zebralogic",
    "score": 0.377,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'ZebraLogic', column 'Qwen3-235B-A22B Non-thinking' = 37.7. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livecodebench-v6",
    "score": 0.329,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table, row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-235B-A22B Non-thinking' = 32.9. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b); non-thinking mode as printed in the header.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau-bench-retail",
    "score": 0.713,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Retail', column 'Qwen3-235B-A22B-Instruct-2507' = 71.3. Own model (non-thinking only, per the card). Added so the row's own-run comparator cells form a run.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-retail",
    "score": 0.652,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Retail', column 'Qwen3-235B-A22B Non-thinking' = 65.2. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b).",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau-bench-retail",
    "score": 0.603,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Retail', column 'GPT-4o-0327', printed '60.3#'. The '#' marks a Qwen-generated result: footnote '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' (Qwen generated these cells itself, with gpt-4o-2024-11-20). Checkpoint therefore gpt-4o-2024-11-20, not the column's GPT-4o-0327.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau-bench-airline",
    "score": 0.44,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Airline', column 'Qwen3-235B-A22B-Instruct-2507' = 44.0. Own model (non-thinking only, per the card). Added so the row's own-run comparator cells form a run.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Airline', column 'Qwen3-235B-A22B Non-thinking' = 32.0. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b).",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau-bench-airline",
    "score": 0.428,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU1-Airline', column 'GPT-4o-0327', printed '42.8#'. The '#' marks a Qwen-generated result: footnote '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' (Qwen generated these cells itself, with gpt-4o-2024-11-20). Checkpoint therefore gpt-4o-2024-11-20, not the column's GPT-4o-0327.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Retail', column 'Qwen3-235B-A22B-Instruct-2507' = 74.6. Own model (non-thinking only, per the card). Added so the row's own-run comparator cells form a run.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-retail",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Retail', column 'Qwen3-235B-A22B Non-thinking' = 64.9. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b).",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-retail",
    "score": 0.667,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Retail', column 'GPT-4o-0327', printed '66.7#'. The '#' marks a Qwen-generated result: footnote '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' (Qwen generated these cells itself, with gpt-4o-2024-11-20). Checkpoint therefore gpt-4o-2024-11-20, not the column's GPT-4o-0327.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.5,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Airline', column 'Qwen3-235B-A22B-Instruct-2507' = 50.0. Own model (non-thinking only, per the card). Added so the row's own-run comparator cells form a run.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-airline",
    "score": 0.36,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Airline', column 'Qwen3-235B-A22B Non-thinking' = 36.0. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b).",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-airline",
    "score": 0.42,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Airline', column 'GPT-4o-0327', printed '42.0#'. The '#' marks a Qwen-generated result: footnote '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' (Qwen generated these cells itself, with gpt-4o-2024-11-20). Checkpoint therefore gpt-4o-2024-11-20, not the column's GPT-4o-0327.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.325,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Telecom', column 'Qwen3-235B-A22B-Instruct-2507' = 32.5. Own model (non-thinking only, per the card). Added so the row's own-run comparator cells form a run.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.246,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Telecom', column 'Qwen3-235B-A22B Non-thinking' = 24.6. Same-family Qwen column 'Qwen3-235B-A22B Non-thinking' in Qwen's own Performance table = the April 2025 hybrid checkpoint Qwen/Qwen3-235B-A22B run with thinking disabled (catalogue qwen3-235b-a22b).",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-telecom",
    "score": 0.298,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507/blob/ac9c66cc9b46af7306746a9250f23d47083d689e/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Instruct-2507 model card @ac9c66c, Performance table (Agent), row 'TAU2-Telecom', column 'GPT-4o-0327', printed '29.8#'. The '#' marks a Qwen-generated result: footnote '#: Results were generated using GPT-4o-20241120, as access to the native function calling API of GPT-4o-0327 was unavailable.' (Qwen generated these cells itself, with gpt-4o-2024-11-20). Checkpoint therefore gpt-4o-2024-11-20, not the column's GPT-4o-0327.",
    "source_id": "card-qwen--qwen3-235b-a22b-instruct-2507"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "mmlu-pro",
    "score": 0.867,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'MMLU-Pro', column 'Qwen3.5-122B-A10B' = 86.7. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "mmlu-pro",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'MMLU-Pro', column 'Qwen3.5-35B-A3B' = 85.3. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.844,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'MMLU-Pro', column 'Qwen3-235B-A22B' = 84.4. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen3-235B-A22B-Thinking-2507: the column's MMLU-Pro 84.4, HLE 18.2, MMLU-Redux 93.8, SuperGPQA 64.9, INCLUDE 81.0 and PolyMATH 60.1 equal the Thinking-2507 figures Qwen published on its own cards (already catalogued), so it is the July 2025 thinking checkpoint, not the April hybrid.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "hle",
    "score": 0.253,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'HLE w/ CoT', column 'Qwen3.5-122B-A10B' = 25.3. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "hle",
    "score": 0.224,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'HLE w/ CoT', column 'Qwen3.5-35B-A3B' = 22.4. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hle",
    "score": 0.182,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'HLE w/ CoT', column 'Qwen3-235B-A22B' = 18.2. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen3-235B-A22B-Thinking-2507: the column's MMLU-Pro 84.4, HLE 18.2, MMLU-Redux 93.8, SuperGPQA 64.9, INCLUDE 81.0 and PolyMATH 60.1 equal the Thinking-2507 figures Qwen published on its own cards (already catalogued), so it is the July 2025 thinking checkpoint, not the April hybrid.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "gpqa-diamond",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'GPQA Diamond', column 'Qwen3.5-122B-A10B' = 86.6. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.842,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'GPQA Diamond', column 'Qwen3.5-35B-A3B' = 84.2. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.811,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'GPQA Diamond', column 'Qwen3-235B-A22B' = 81.1. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen3-235B-A22B-Thinking-2507: the column's MMLU-Pro 84.4, HLE 18.2, MMLU-Redux 93.8, SuperGPQA 64.9, INCLUDE 81.0 and PolyMATH 60.1 equal the Thinking-2507 figures Qwen published on its own cards (already catalogued), so it is the July 2025 thinking checkpoint, not the April hybrid.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "swe-bench-verified",
    "score": 0.72,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'SWE-bench Verified', column 'Qwen3.5-122B-A10B' = 72.0. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.692,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'SWE-bench Verified', column 'Qwen3.5-35B-A3B' = 69.2. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "livecodebench-v6",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3.5-122B-A10B' = 78.9. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3.5-35B-A3B' = 74.6. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.751,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3-235B-A22B' = 75.1. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen3-235B-A22B-Thinking-2507: the column's MMLU-Pro 84.4, HLE 18.2, MMLU-Redux 93.8, SuperGPQA 64.9, INCLUDE 81.0 and PolyMATH 60.1 equal the Thinking-2507 figures Qwen published on its own cards (already catalogued), so it is the July 2025 thinking checkpoint, not the April hybrid.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "tau2-bench",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'TAU2-Bench', column 'Qwen3.5-27B' = 79.0. Qwen's own run under the card's setup (footnote '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.'). Checkpoint: own model. New run (own Qwen3.5-27B TAU2 cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "tau2-bench",
    "score": 0.795,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'TAU2-Bench', column 'Qwen3.5-122B-A10B' = 79.5. Qwen's own run under the card's setup (footnote '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.'). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release. New run (own Qwen3.5-27B TAU2 cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "tau2-bench",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'TAU2-Bench', column 'Qwen3.5-35B-A3B' = 81.2. Qwen's own run under the card's setup (footnote '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.'). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release. New run (own Qwen3.5-27B TAU2 cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-bench",
    "score": 0.585,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'TAU2-Bench', column 'Qwen3-235B-A22B' = 58.5. Qwen's own run under the card's setup (footnote '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.'). Checkpoint: Qwen3-235B-A22B-Thinking-2507: the column's MMLU-Pro 84.4, HLE 18.2, MMLU-Redux 93.8, SuperGPQA 64.9, INCLUDE 81.0 and PolyMATH 60.1 equal the Thinking-2507 figures Qwen published on its own cards (already catalogued), so it is the July 2025 thinking checkpoint, not the April hybrid. New run (own Qwen3.5-27B TAU2 cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-bench",
    "score": 0.698,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "independent",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Language benchmark table, row 'TAU2-Bench', column 'GPT-5-mini 2025-08-07' = 69.8. Comparator evaluated by Qwen: footnote '* TAU2-Bench: we follow the official setup except for the airline domain, where all models are evaluated by applying the fixes proposed in the Claude Opus 4.5 system card.'. Checkpoint: gpt-5-mini-2025-08-07 (dated id printed in the header); reasoning effort not printed.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "math-vision",
    "score": 0.86,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Vision Language benchmark table, row 'MathVision', column 'Qwen3.5-27B' = 86.0. Qwen's own evaluation (footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.'). Checkpoint: own model. New run (own MathVision cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "math-vision",
    "score": 0.862,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Vision Language benchmark table, row 'MathVision', column 'Qwen3.5-122B-A10B' = 86.2. Qwen's own evaluation (footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.'). Checkpoint: Qwen/Qwen3.5-122B-A10B open-weights release. New run (own MathVision cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "math-vision",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "lab",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Vision Language benchmark table, row 'MathVision', column 'Qwen3.5-35B-A3B' = 83.9. Qwen's own evaluation (footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.'). Checkpoint: Qwen/Qwen3.5-35B-A3B open-weights release. New run (own MathVision cell not imported before).",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "gpt-5-mini",
    "bench": "math-vision",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "independent",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Vision Language benchmark table, row 'MathVision', column 'GPT-5-mini 2025-08-07' = 71.9. Comparator run by Qwen: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.'. Checkpoint: gpt-5-mini-2025-08-07; reasoning effort not printed.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "math-vision",
    "score": 0.711,
    "source": "https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md",
    "by": "independent",
    "date": "2026-02-24",
    "note": "Qwen3.5-27B model card @fc05dae, Vision Language benchmark table, row 'MathVision', column 'Claude-Sonnet-4.5' = 71.1. Comparator run by Qwen: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.'. Checkpoint: Claude Sonnet 4.5 (single release, claude-sonnet-4-5-20250929); thinking setting not printed.",
    "source_id": "card-qwen--qwen3.5-27b"
  },
  {
    "model": "deepseek-r1",
    "bench": "mmlu-redux",
    "score": 0.929,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'MMLU-Redux (EM)', section 'English', column 'DeepSeek R1' = 92.9. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "simpleqa",
    "score": 0.301,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'SimpleQA (Correct)', section 'English', column 'DeepSeek R1' = 30.1. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "frames",
    "score": 0.825,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'FRAMES (Acc.)', section 'English', column 'DeepSeek R1' = 82.5. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "aider-polyglot",
    "score": 0.533,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'Aider-Polyglot (Acc.)', section 'Code', column 'DeepSeek R1' = 53.3. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "cnmo-2024",
    "score": 0.788,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'CNMO 2024 (Pass@1)', section 'Math', column 'DeepSeek R1' = 78.8. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2024",
    "score": 0.798,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek own evaluation in the pinned DeepSeek-R1 model card @56d4cbb, DeepSeek-R1-Evaluation table, row 'AIME 2024 (Pass@1)', section 'Math', column 'DeepSeek R1' = 79.8. 'For all our models, the maximum generation length is set to 32,768 tokens. For benchmarks requiring sampling, we use a temperature of 0.6, a top-p value of 0.95, and generate 64 responses per query to estimate pass@1.' Own model (original January 2025 DeepSeek-R1). The first import skipped this cell because the same number was already recorded from the DeepSeek-R1-0528 card; without it the DeepSeek V3 cell of this row is alone in its run.",
    "source_id": "card-deepseek-ai--deepseek-r1"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.595,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'MMLU-ProX', column 'Nemotron-3-Nano 30B-A3B', printed '59.5'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "mmlu-prox",
    "score": 0.794,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'MMLU-ProX', column 'Nemotron-3-Super 120B-A12B', printed '79.4'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "multichallenge",
    "score": 0.385,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'Scale AI Multi-Challenge', column 'Nemotron-3-Nano 30B-A3B', printed '38.5'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "multichallenge",
    "score": 0.552,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'Scale AI Multi-Challenge', column 'Nemotron-3-Super 120B-A12B', printed '55.2'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "scicode-unspecified",
    "score": 0.333,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'SciCode', column 'Nemotron-3-Nano 30B-A3B', printed '33.3'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "scicode-unspecified",
    "score": 0.421,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'SciCode', column 'Nemotron-3-Super 120B-A12B', printed '42.1'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "aime-2025",
    "score": 0.891,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2025', column 'Nemotron-3-Nano 30B-A3B', printed '89.1'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "aime-2025",
    "score": 0.902,
    "source": "https://arxiv.org/html/2603.19220v2",
    "by": "lab",
    "date": "2026-03-22",
    "note": "Nemotron-Cascade 2 report (NVIDIA, arXiv 2603.19220v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2025', column 'Nemotron-3-Super 120B-A12B', printed '90.2'. Same-lab NVIDIA model in NVIDIA's own Table 1. Checkpoint: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B.",
    "source_id": "report-2603.19220v2"
  },
  {
    "model": "glm-5-1",
    "bench": "hle",
    "score": 0.31,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'HLE', column 'GLM-5.1' = 31. Same-lab previous model in Z.ai's own table (settings as the GLM-5.2 cell: 'temperature=1.0, top_p=0.95 ... maximum generation length of 163,840 tokens'). Text-only subset (footnote: 'By default, we report the text-only subset; results marked with * are from the full set'; this cell is unmarked). Checkpoint: zai-org/GLM-5.1.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-1",
    "bench": "aime-2026",
    "score": 0.953,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'AIME 2026', column 'GLM-5.1' = 95.3. Same-lab previous model in Z.ai's own table (settings as the GLM-5.2 cell: 'temperature=1.0, top_p=0.95 ... maximum generation length of 163,840 tokens').  Checkpoint: zai-org/GLM-5.1.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-1",
    "bench": "gpqa-diamond",
    "score": 0.862,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'GPQA-Diamond', column 'GLM-5.1' = 86.2. Same-lab previous model in Z.ai's own table (settings as the GLM-5.2 cell: 'temperature=1.0, top_p=0.95 ... maximum generation length of 163,840 tokens').  Checkpoint: zai-org/GLM-5.1.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-pro",
    "score": 0.584,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'SWE-bench Pro', column 'GLM-5.1' = 58.4. Same-lab previous model in Z.ai's own table (settings as the GLM-5.2 cell: 'temperature=1.0, top_p=0.95 ... maximum generation length of 163,840 tokens'). Footnote: 'We run the SWE-Bench Pro suite with OpenHands using a tailored instruction prompt.' Checkpoint: zai-org/GLM-5.1.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "qwen3-7-max",
    "bench": "mcp-atlas",
    "score": 0.764,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'Qwen3.7-Max' = 76.4. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: Qwen3.7 Max (GA May 2026).",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "minimax-m3",
    "bench": "mcp-atlas",
    "score": 0.742,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'MiniMax M3' = 74.2. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: MiniMaxAI/MiniMax-M3.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "mcp-atlas",
    "score": 0.736,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'DeepSeek-V4-Pro' = 73.6. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: the April 2026 DeepSeek-V4-Pro release (deepseek-ai/DeepSeek-V4-Pro): the table was already unchanged at the 2026-07-02 revision b4734de (diff shows only a 'new_version' metadata line added in cf457fa), before the 0813 checkpoint existed.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "mcp-atlas",
    "score": 0.778,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'Claude Opus 4.8' = 77.8. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: Claude Opus 4.8.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "gpt-5-5",
    "bench": "mcp-atlas",
    "score": 0.753,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'GPT-5.5' = 75.3. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: GPT-5.5; reasoning effort not printed beyond think mode.",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "mcp-atlas",
    "score": 0.692,
    "source": "https://huggingface.co/zai-org/GLM-5.2/blob/cf457fa734ab149ffef225f80893eb38c6ff5cdc/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "GLM-5.2 model card @cf457fa, Benchmark table, row 'MCP-Atlas (Public Set)', column 'Gemini 3.1 Pro' = 69.2. Run by Z.ai under one setup: Footnote: '* MCP-Atlas: All models were evaluated in think mode on the 500-task public subset with a 10-minute timeout per task. We use Gemini-3.0-Pro as the judge model for evaluation.' Checkpoint: Gemini 3.1 Pro (Preview).",
    "source_id": "card-zai-org--glm-5.2"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "mmlu-pro",
    "score": 0.827,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'MMLU-Pro', column 'Qwen3-Next-80B-A3B-Thinking' = 82.7. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-Next-80B-A3B-Thinking.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.809,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'MMLU-Pro', column 'Qwen3-30BA3B-Thinking-2507' = 80.9. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-pro",
    "score": 0.825,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'MMLU-Pro', column 'Qwen3.5-9B' = 82.5. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-9B.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.772,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'GPQA Diamond', column 'Qwen3-Next-80B-A3B-Thinking' = 77.2. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-Next-80B-A3B-Thinking.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.734,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'GPQA Diamond', column 'Qwen3-30BA3B-Thinking-2507' = 73.4. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.817,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'GPQA Diamond', column 'Qwen3.5-9B' = 81.7. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-9B.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "livecodebench-v6",
    "score": 0.687,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3-Next-80B-A3B-Thinking' = 68.7. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-Next-80B-A3B-Thinking.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "livecodebench-v6",
    "score": 0.66,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3-30BA3B-Thinking-2507' = 66.0. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3-30B-A3B-Thinking-2507.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6",
    "score": 0.656,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Language benchmark table, row 'LiveCodeBench v6', column 'Qwen3.5-9B' = 65.6. Same-family Qwen column in Qwen's own table (thinking mode default). Checkpoint: Qwen/Qwen3.5-9B.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "math-vision",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Vision Language benchmark table, row 'MathVision', column 'Qwen3.5-4B' = 74.6. Qwen's own evaluation: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: own model (not imported before). New run from the same card.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "math-vision",
    "score": 0.789,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "lab",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Vision Language benchmark table, row 'MathVision', column 'Qwen3.5-9B' = 78.9. Qwen's own evaluation: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: Qwen/Qwen3.5-9B. New run from the same card.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "gpt-5-nano",
    "bench": "math-vision",
    "score": 0.622,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "independent",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Vision Language benchmark table, row 'MathVision', column 'GPT-5-Nano-2025-08-07' = 62.2. Comparator run by Qwen: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: gpt-5-nano-2025-08-07 (dated id in header); reasoning effort not printed. New run from the same card.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "gemini-2-5-flash-lite",
    "bench": "math-vision",
    "score": 0.521,
    "source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
    "by": "independent",
    "date": "2026-03-02",
    "note": "Qwen3.5-4B model card @851bf6e, Vision Language benchmark table, row 'MathVision', column 'Gemini-2.5-Flash-Lite' = 52.1. Comparator run by Qwen: footnote '* MathVision: our model’s score is evaluated using a fixed prompt ... For other models, we report the higher score between runs with and without the \\boxed{} formatting.' Checkpoint: GA gemini-2.5-flash-lite (the card names no preview date); thinking setting not printed. New run from the same card.",
    "source_id": "card-qwen--qwen3.5-4b"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.661,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'Gemini 2.5 Pro', vanilla value '66.1±1.0' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Gemini 2.5 Pro via Google API (paper cites the GA Gemini 2.5 Pro model card; no dated id).",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.572,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'Claude Opus 4.1*', vanilla value '57.2±1.0' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-opus-4-1-20250805-v1:0, thinking budget 8k.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.549,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'Claude Sonnet 4.5', vanilla value '54.9±0.7' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.anthropic.claude-sonnet-4-5-20250929-v1:0, thinking budget 8k.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-5",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.539,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'GPT-5', vanilla value '53.9±1.1' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-5-2025-08-07, reasoning effort medium.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.504,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'Claude 3.7 Sonnet', vanilla value '50.4±1.0' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.anthropic.claude-3-7-sonnet-20250219-v1:0, thinking budget 8k.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "o3",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.502,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'o3', vanilla value '50.2±1.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: o3-2025-04-16, reasoning effort medium.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.457,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large Reasoning', row 'Claude Sonnet 4*', vanilla value '45.7±2.1' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-sonnet-4-20250514-v1:0, thinking budget 8k.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.518,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small Reasoning', row 'Gemini 2.5 Flash', vanilla value '51.8±0.6' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Gemini 2.5 Flash via Google API (GA model card cited; no dated id).",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-5-mini",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.409,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small Reasoning', row 'GPT-5 Mini', vanilla value '40.9±1.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-5-mini-2025-08-07, reasoning effort medium.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "o4-mini",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.377,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small Reasoning', row 'o4-mini', vanilla value '37.7±1.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: o4-mini-2025-04-16, reasoning effort medium.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "qwen3-32b",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.35,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small Reasoning', row 'Qwen3 32B', vanilla value '35.0±3.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Qwen/Qwen3-32B (hybrid) in the Small Reasoning group, run with vLLM.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.292,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small Reasoning', row 'gpt-oss 120B', vanilla value '29.2±1.6' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: openai/gpt-oss-120b, run with vLLM; reasoning level not printed.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4-1",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.479,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large General', row 'GPT-4.1', vanilla value '47.9±1.0' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4.1-2025-04-14.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.451,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large General', row 'Claude 3.5 Sonnet*', vanilla value '45.1±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-3-5-sonnet-20240620-v1:0.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "llama-4-maverick",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.433,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large General', row 'Llama 4 Maverick', vanilla value '43.3±0.4' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.meta.llama4-maverick-17b-instruct-v1:0.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.433,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large General', row 'GPT-4o', vanilla value '43.3±2.0' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4o-2024-08-06.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "deepseek-v3",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.373,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Large General', row 'DeepSeek-V3', vanilla value '37.3±0.7' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: original DeepSeek-V3 (Appendix: 'DeepSeek-V3 ... is available under DEEPSEEK LICENSE', the Dec-2024 release's model licence; V3-0324 is MIT), run with vLLM.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gemma-3-27b",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.351,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'Gemma 3 27B', vanilla value '35.1±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: google/gemma-3-27b-it, run with vLLM.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.299,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'GPT-4o-mini', vanilla value '29.9±1.5' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: GPT-4o-mini via OpenAI API (gpt-4o-mini-2024-07-18 is its only snapshot).",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.294,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'GPT-4.1-mini', vanilla value '29.4±0.7' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4.1-mini-2025-04-14.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "aya-expanse-32b",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.284,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'Aya Expanse 32B', vanilla value '28.4±0.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: CohereLabs/aya-expanse-32b, run with vLLM.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.28,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'Claude 3.5 Haiku*', vanilla value '28.0±1.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-3-5-haiku-20241022-v1:0.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "phi-4-mini",
    "bench": "kcl-mcqa-vanilla",
    "score": 0.22,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 2 (KCL-MCQA), group 'Small General', row 'Phi-4-mini', vanilla value '22.0±0.8' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: microsoft/Phi-4-mini-instruct, run with vLLM.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "kcl-essay-vanilla",
    "score": 0.605,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'Gemini 2.5 Pro', vanilla value '60.5±0.5' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Gemini 2.5 Pro via Google API (paper cites the GA Gemini 2.5 Pro model card; no dated id). Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "kcl-essay-vanilla",
    "score": 0.455,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'Claude Opus 4.1*', vanilla value '45.5±0.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-opus-4-1-20250805-v1:0, thinking budget 8k. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "kcl-essay-vanilla",
    "score": 0.476,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'Claude Sonnet 4.5', vanilla value '47.6±1.5' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.anthropic.claude-sonnet-4-5-20250929-v1:0, thinking budget 8k. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-5",
    "bench": "kcl-essay-vanilla",
    "score": 0.575,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'GPT-5', vanilla value '57.5±1.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-5-2025-08-07, reasoning effort medium. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "kcl-essay-vanilla",
    "score": 0.436,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'Claude 3.7 Sonnet', vanilla value '43.6±0.7' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.anthropic.claude-3-7-sonnet-20250219-v1:0, thinking budget 8k. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "o3",
    "bench": "kcl-essay-vanilla",
    "score": 0.512,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'o3', vanilla value '51.2±0.6' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: o3-2025-04-16, reasoning effort medium. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "kcl-essay-vanilla",
    "score": 0.355,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large Reasoning', row 'Claude Sonnet 4*', vanilla value '35.5±1.2' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-sonnet-4-20250514-v1:0, thinking budget 8k. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "kcl-essay-vanilla",
    "score": 0.537,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small Reasoning', row 'Gemini 2.5 Flash', vanilla value '53.7±1.1' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Gemini 2.5 Flash via Google API (GA model card cited; no dated id). Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-5-mini",
    "bench": "kcl-essay-vanilla",
    "score": 0.403,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small Reasoning', row 'GPT-5 Mini', vanilla value '40.3±0.5' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-5-mini-2025-08-07, reasoning effort medium. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "o4-mini",
    "bench": "kcl-essay-vanilla",
    "score": 0.313,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small Reasoning', row 'o4-mini', vanilla value '31.3±0.9' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: o4-mini-2025-04-16, reasoning effort medium. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "qwen3-32b",
    "bench": "kcl-essay-vanilla",
    "score": 0.292,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small Reasoning', row 'Qwen3 32B', vanilla value '29.2±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Qwen/Qwen3-32B (hybrid) in the Small Reasoning group, run with vLLM. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "kcl-essay-vanilla",
    "score": 0.316,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small Reasoning', row 'gpt-oss 120B', vanilla value '31.6±0.5' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: openai/gpt-oss-120b, run with vLLM; reasoning level not printed. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4-1",
    "bench": "kcl-essay-vanilla",
    "score": 0.424,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large General', row 'GPT-4.1', vanilla value '42.4±0.8' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4.1-2025-04-14. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "kcl-essay-vanilla",
    "score": 0.337,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large General', row 'Claude 3.5 Sonnet*', vanilla value '33.7±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-3-5-sonnet-20240620-v1:0. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "llama-4-maverick",
    "bench": "kcl-essay-vanilla",
    "score": 0.256,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large General', row 'Llama 4 Maverick', vanilla value '25.6±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: Bedrock us.meta.llama4-maverick-17b-instruct-v1:0. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "kcl-essay-vanilla",
    "score": 0.243,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large General', row 'GPT-4o', vanilla value '24.3±0.4' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4o-2024-08-06. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "deepseek-v3",
    "bench": "kcl-essay-vanilla",
    "score": 0.343,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Large General', row 'DeepSeek-V3', vanilla value '34.3±0.6' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: original DeepSeek-V3 (Appendix: 'DeepSeek-V3 ... is available under DEEPSEEK LICENSE', the Dec-2024 release's model licence; V3-0324 is MIT), run with vLLM. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gemma-3-27b",
    "bench": "kcl-essay-vanilla",
    "score": 0.236,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'Gemma 3 27B', vanilla value '23.6±0.4' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: google/gemma-3-27b-it, run with vLLM. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "kcl-essay-vanilla",
    "score": 0.161,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'GPT-4o-mini', vanilla value '16.1±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: GPT-4o-mini via OpenAI API (gpt-4o-mini-2024-07-18 is its only snapshot). Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "kcl-essay-vanilla",
    "score": 0.309,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'GPT-4.1-mini', vanilla value '30.9±0.7' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: gpt-4.1-mini-2025-04-14. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "aya-expanse-32b",
    "bench": "kcl-essay-vanilla",
    "score": 0.202,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'Aya Expanse 32B', vanilla value '20.2±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: CohereLabs/aya-expanse-32b, run with vLLM. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "kcl-essay-vanilla",
    "score": 0.212,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'Claude 3.5 Haiku*', vanilla value '21.2±0.6' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. '*' marks truncation of over-long precedents, which affects only the w/ precedents setting, not this vanilla value. Checkpoint: Bedrock us.anthropic.claude-3-5-haiku-20241022-v1:0. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "phi-4-mini",
    "bench": "kcl-essay-vanilla",
    "score": 0.06,
    "source": "https://aclanthology.org/2026.eacl-short.17.pdf",
    "by": "independent",
    "date": "2026-09-25",
    "note": "KCL paper (Oh, Hwang and On, EACL 2026 short, CC BY 4.0), Table 3 (KCL-Essay), group 'Small General', row 'Phi-4-mini', vanilla value '6.0±0.3' (mean of three runs, no supporting precedents). §3: 'We used vLLM (Kwon et al., 2023) to evaluate open-weight models, and the usage details for open-API models are provided in Appendix C'; §4.1: 'We evaluated various LLMs on the vanilla setting first'. Every row is the KCL authors' own run. Checkpoint: microsoft/Phi-4-mini-instruct, run with vLLM. Essay judged by Gemini 2.5 Flash with instance-level rubrics.",
    "source_id": "acl-kcl-2026"
  },
  {
    "model": "olmo-3-7b-think-dpo",
    "bench": "math-subset-unspecified",
    "score": 0.924,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442, Evaluation table, row 'MATH', column 'Olmo 3 Think 7B DPO' = 92.4. Ai2's own intermediate checkpoint in Ai2's own table (the card's stage table links it as the DPO stage of Olmo 3 7B Think); same row/settings as the final Olmo 3 Think 7B cell ('We evaluated our models on the following settings ... temperature 0.6, top_p 0.95, max_tokens 32768'). Checkpoint: allenai/Olmo-3-7B-Think-DPO (proposed comparison-only model).",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "olmo-3-7b-think-dpo",
    "bench": "omega",
    "score": 0.405,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442, Evaluation table, row 'OMEGA', column 'Olmo 3 Think 7B DPO' = 40.5. Ai2's own intermediate checkpoint in Ai2's own table (the card's stage table links it as the DPO stage of Olmo 3 7B Think); same row/settings as the final Olmo 3 Think 7B cell ('We evaluated our models on the following settings ... temperature 0.6, top_p 0.95, max_tokens 32768'). Checkpoint: allenai/Olmo-3-7B-Think-DPO (proposed comparison-only model).",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "ministral-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.876,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral own evaluation in the pinned Ministral-3-14B-Instruct-2512 model card @29439f8, 'Instruct' benchmark table, row 'Ministral 3 8B', column 'MATH Maj@1' = 0.876 (already a fraction). Same-family Instruct 2512 checkpoint in Mistral's own table, same row as the 14B cell (the earlier import used the same rows for Arena Hard). Full MATH vs MATH-500 not stated. Checkpoint: mistralai/Ministral-3-8B-Instruct-2512.",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512"
  },
  {
    "model": "ministral-3-3b",
    "bench": "math-subset-unspecified",
    "score": 0.83,
    "source": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/blob/29439f81c2be264d8d393273f99e7db9c0961120/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Mistral own evaluation in the pinned Ministral-3-14B-Instruct-2512 model card @29439f8, 'Instruct' benchmark table, row 'Ministral 3 3B', column 'MATH Maj@1' = 0.830 (already a fraction). Same-family Instruct 2512 checkpoint in Mistral's own table, same row as the 14B cell (the earlier import used the same rows for Arena Hard). Full MATH vs MATH-500 not stated. Checkpoint: mistralai/Ministral-3-3B-Instruct-2512.",
    "source_id": "card-mistralai-ministral-3-14b-instruct-2512"
  },
  {
    "model": "command-a-plus",
    "bench": "mmlu-pro",
    "score": 0.79,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'Command A+ 218B-A25B' = 79.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MMLU-Pro (Wang et al., 2024b). Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "mmlu-pro",
    "score": 0.812,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'Mistral Medium 3.5 128B dense, high' = 81.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MMLU-Pro (Wang et al., 2024b). Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "mmlu-pro",
    "score": 0.846,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'MiMo-V2.5 310B-A15B' = 84.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MMLU-Pro (Wang et al., 2024b). Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "mmlu-pro",
    "score": 0.859,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'DeepSeek-V4-Flash 284B-A13B, max' = 85.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MMLU-Pro (Wang et al., 2024b). Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "gpqa-diamond",
    "score": 0.756,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'Command A+ 218B-A25B' = 75.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' GPQA-Diamond (Rein et al., 2024). Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "gpqa-diamond",
    "score": 0.775,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'Mistral Medium 3.5 128B dense, high' = 77.5. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' GPQA-Diamond (Rein et al., 2024). Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "gpqa-diamond",
    "score": 0.83,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'MiMo-V2.5 310B-A15B' = 83.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' GPQA-Diamond (Rein et al., 2024). Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "gpqa-diamond",
    "score": 0.889,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'DeepSeek-V4-Flash 284B-A13B, max' = 88.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' GPQA-Diamond (Rein et al., 2024). Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "hle",
    "score": 0.114,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'Command A+ 218B-A25B' = 11.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HLE without tools'; text-only vs full set not stated. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "hle",
    "score": 0.128,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'Mistral Medium 3.5 128B dense, high' = 12.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HLE without tools'; text-only vs full set not stated. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "hle",
    "score": 0.243,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'MiMo-V2.5 310B-A15B' = 24.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HLE without tools'; text-only vs full set not stated. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "hle",
    "score": 0.323,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'DeepSeek-V4-Flash 284B-A13B, max' = 32.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HLE without tools'; text-only vs full set not stated. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.861,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'Command A+ 218B-A25B' = 86.1. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'LiveCodeBench v6'; date window not stated, so the window-unspecified id. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.849,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'Mistral Medium 3.5 128B dense, high' = 84.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'LiveCodeBench v6'; date window not stated, so the window-unspecified id. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.891,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'MiMo-V2.5 310B-A15B' = 89.1. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'LiveCodeBench v6'; date window not stated, so the window-unspecified id. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.923,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'DeepSeek-V4-Flash 284B-A13B, max' = 92.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'LiveCodeBench v6'; date window not stated, so the window-unspecified id. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "hmmt-feb-2026",
    "score": 0.735,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'Command A+ 218B-A25B' = 73.5. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HMMT2602 ... from MathArena' = HMMT February 2026. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "hmmt-feb-2026",
    "score": 0.629,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'Mistral Medium 3.5 128B dense, high' = 62.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HMMT2602 ... from MathArena' = HMMT February 2026. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "hmmt-feb-2026",
    "score": 0.614,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'MiMo-V2.5 310B-A15B' = 61.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HMMT2602 ... from MathArena' = HMMT February 2026. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "hmmt-feb-2026",
    "score": 0.947,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'DeepSeek-V4-Flash 284B-A13B, max' = 94.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'HMMT2602 ... from MathArena' = HMMT February 2026. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "aime-2026",
    "score": 0.96,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'Command A+ 218B-A25B' = 96.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'AIME2026 ... from MathArena'; I and II not split, so aime-2026. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "aime-2026",
    "score": 0.89,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'Mistral Medium 3.5 128B dense, high' = 89.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'AIME2026 ... from MathArena'; I and II not split, so aime-2026. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "aime-2026",
    "score": 0.923,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'MiMo-V2.5 310B-A15B' = 92.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'AIME2026 ... from MathArena'; I and II not split, so aime-2026. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "aime-2026",
    "score": 0.97,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'DeepSeek-V4-Flash 284B-A13B, max' = 97.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'AIME2026 ... from MathArena'; I and II not split, so aime-2026. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "multichallenge",
    "score": 0.458,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'Command A+ 218B-A25B' = 45.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Multi-Challenge (Sirdeshmukh et al., 2025); judge not stated. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "multichallenge",
    "score": 0.498,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'Mistral Medium 3.5 128B dense, high' = 49.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Multi-Challenge (Sirdeshmukh et al., 2025); judge not stated. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "multichallenge",
    "score": 0.39,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'MiMo-V2.5 310B-A15B' = 39.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Multi-Challenge (Sirdeshmukh et al., 2025); judge not stated. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "multichallenge",
    "score": 0.62,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'DeepSeek-V4-Flash 284B-A13B, max' = 62.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Multi-Challenge (Sirdeshmukh et al., 2025); judge not stated. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "ifbench-unspecified",
    "score": 0.739,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'Command A+ 218B-A25B' = 73.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' IFBench (Pyatkin et al., 2025); prompt/instruction-level aggregation not stated, so ifbench-unspecified. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "ifbench-unspecified",
    "score": 0.69,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'Mistral Medium 3.5 128B dense, high' = 69.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' IFBench (Pyatkin et al., 2025); prompt/instruction-level aggregation not stated, so ifbench-unspecified. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "ifbench-unspecified",
    "score": 0.671,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'MiMo-V2.5 310B-A15B' = 67.1. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' IFBench (Pyatkin et al., 2025); prompt/instruction-level aggregation not stated, so ifbench-unspecified. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "ifbench-unspecified",
    "score": 0.803,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'DeepSeek-V4-Flash 284B-A13B, max' = 80.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' IFBench (Pyatkin et al., 2025); prompt/instruction-level aggregation not stated, so ifbench-unspecified. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "swe-bench-verified",
    "score": 0.144,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'Command A+ 218B-A25B' = 14.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' SWE-Bench Verified (Jimenez et al., 2023); agent scaffold not stated (internal harness). Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "swe-bench-verified",
    "score": 0.696,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'Mistral Medium 3.5 128B dense, high' = 69.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' SWE-Bench Verified (Jimenez et al., 2023); agent scaffold not stated (internal harness). Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "swe-bench-verified",
    "score": 0.73,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'MiMo-V2.5 310B-A15B' = 73.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' SWE-Bench Verified (Jimenez et al., 2023); agent scaffold not stated (internal harness). Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "swe-bench-verified",
    "score": 0.738,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'DeepSeek-V4-Flash 284B-A13B, max' = 73.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' SWE-Bench Verified (Jimenez et al., 2023); agent scaffold not stated (internal harness). Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.25,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'Command A+ 218B-A25B' = 25.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'Terminal Bench Hard (Merrill et al., 2026)' in the Agents group next to AA-LCR and GDPval-AA (the Artificial Analysis index suite); task count, agent harness and repeats are not printed. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.333,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'Mistral Medium 3.5 128B dense, high' = 33.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'Terminal Bench Hard (Merrill et al., 2026)' in the Agents group next to AA-LCR and GDPval-AA (the Artificial Analysis index suite); task count, agent harness and repeats are not printed. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.417,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'MiMo-V2.5 310B-A15B' = 41.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'Terminal Bench Hard (Merrill et al., 2026)' in the Agents group next to AA-LCR and GDPval-AA (the Artificial Analysis index suite); task count, agent harness and repeats are not printed. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.341,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'DeepSeek-V4-Flash 284B-A13B, max' = 34.1. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'Terminal Bench Hard (Merrill et al., 2026)' in the Agents group next to AA-LCR and GDPval-AA (the Artificial Analysis index suite); task count, agent harness and repeats are not printed. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "apex-agents",
    "score": 0.016,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'Command A+ 218B-A25B' = 1.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' APEX-Agents (Vidgen et al., 2026); pass@k not printed (pass@1 assumed by the id's definition 'as reported'). Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "apex-agents",
    "score": 0.061,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'Mistral Medium 3.5 128B dense, high' = 6.1. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' APEX-Agents (Vidgen et al., 2026); pass@k not printed (pass@1 assumed by the id's definition 'as reported'). Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "apex-agents",
    "score": 0.134,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'MiMo-V2.5 310B-A15B' = 13.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' APEX-Agents (Vidgen et al., 2026); pass@k not printed (pass@1 assumed by the id's definition 'as reported'). Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "apex-agents",
    "score": 0.132,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'DeepSeek-V4-Flash 284B-A13B, max' = 13.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' APEX-Agents (Vidgen et al., 2026); pass@k not printed (pass@1 assumed by the id's definition 'as reported'). Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "mcp-atlas",
    "score": 0.272,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'Command A+ 218B-A25B' = 27.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MCP-Atlas (Bandi et al., 2026); split/judge not printed, so mcp-atlas (public set, harness per note), not mcp-atlas-v1-500. Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "mcp-atlas",
    "score": 0.307,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'Mistral Medium 3.5 128B dense, high' = 30.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MCP-Atlas (Bandi et al., 2026); split/judge not printed, so mcp-atlas (public set, harness per note), not mcp-atlas-v1-500. Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "mcp-atlas",
    "score": 0.639,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'MiMo-V2.5 310B-A15B' = 63.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MCP-Atlas (Bandi et al., 2026); split/judge not printed, so mcp-atlas (public set, harness per note), not mcp-atlas-v1-500. Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "mcp-atlas",
    "score": 0.582,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'DeepSeek-V4-Flash 284B-A13B, max' = 58.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' MCP-Atlas (Bandi et al., 2026); split/judge not printed, so mcp-atlas (public set, harness per note), not mcp-atlas-v1-500. Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "command-a-plus",
    "bench": "tau3-banking",
    "score": 0.058,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'Command A+ 218B-A25B' = 5.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'the τ3 banking domain of the τ-bench suite'; tau2-bench grading version not printed, so tau3-banking (not tau3-banking-v101). Header 'Command A+ / 218B-A25B' = Cohere command-a-plus-05-2026 (218B; the only Command A+ release, May 2026). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "tau3-banking",
    "score": 0.058,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'Mistral Medium 3.5 128B dense, high' = 5.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'the τ3 banking domain of the τ-bench suite'; tau2-bench grading version not printed, so tau3-banking (not tau3-banking-v101). Header 'Mistral Medium 3.5 / 128B dense, high' = catalogue mistral-medium-3-5 (128B dense); reasoning effort 'high' as printed. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "mimo-v2-5",
    "bench": "tau3-banking",
    "score": 0.087,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'MiMo-V2.5 310B-A15B' = 8.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'the τ3 banking domain of the τ-bench suite'; tau2-bench grading version not printed, so tau3-banking (not tau3-banking-v101). Header 'MiMo-V2.5 / 310B-A15B' = catalogue mimo-v2-5 (XiaomiMiMo/MiMo-V2.5, 310B-A15B). Reasoning setting not printed for this column. Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "tau3-banking",
    "score": 0.223,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "independent",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'DeepSeek-V4-Flash 284B-A13B, max' = 22.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' 'the τ3 banking domain of the τ-bench suite'; tau2-bench grading version not printed, so tau3-banking (not tau3-banking-v101). Header 'DeepSeek-V4-Flash / 284B-A13B, max' with reasoning effort 'max'. Report v1 is dated 2026-07-22 and v2 2026-07-24, before DeepSeek-V4-Flash-0731 existed, so this is the April 2026 preview weights (catalogue deepseek-v4-flash-preview), not deepseek-v4-flash (0731). Upstage's own Solar Open 2 / Solar Open 1 columns are not catalogue models (see gaps). Table identical in v1 (2026-07-22) and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "falcon-h1r-7b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.049,
    "source": "https://arxiv.org/html/2601.02346v1#S4.T5",
    "by": "lab",
    "date": "2026-01-05",
    "note": "Falcon-H1R report (TII, arXiv 2601.02346v1, CC BY 4.0), Table 5 'Code generation with reasoning evaluations', column 'TB Hard', row 'Falcon-H1R-7B' = 4.9. Sec. 4.1: 'Terminal-Bench (TB) Hard evaluates 47 hard tasks from the terminal-bench-core dataset (commit 74221fb) using the Terminus 2 agent. Each task is capped at 100 episodes and a 24-hour timeout, with model configurations set to the framework's default values'; 'For TB Hard ... we repeat each evaluation three times and report the average result'; 3 responses per query, max response length 32768. Baselines: 'we evaluated a range of SOTA models ... using each baseline's recommended evaluation configurations'; cells marked * are 'taken from Artificial Analysis' (Apriel-1.5-15b-Thinker 9.9* and GPT-OSS-20B 9.9* excluded); this cell is unstarred, i.e. the authors' own run. Authors' own model; sampling temperature 0.6, top-p 0.95 with the report's system prompt (Sec. 4.1). = tiiuae/Falcon-H1R-7B.",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-8b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.021,
    "source": "https://arxiv.org/html/2601.02346v1#S4.T5",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R report (TII, arXiv 2601.02346v1, CC BY 4.0), Table 5 'Code generation with reasoning evaluations', column 'TB Hard', row 'Qwen3-8B' = 2.1. Sec. 4.1: 'Terminal-Bench (TB) Hard evaluates 47 hard tasks from the terminal-bench-core dataset (commit 74221fb) using the Terminus 2 agent. Each task is capped at 100 episodes and a 24-hour timeout, with model configurations set to the framework's default values'; 'For TB Hard ... we repeat each evaluation three times and report the average result'; 3 responses per query, max response length 32768. Baselines: 'we evaluated a range of SOTA models ... using each baseline's recommended evaluation configurations'; cells marked * are 'taken from Artificial Analysis' (Apriel-1.5-15b-Thinker 9.9* and GPT-OSS-20B 9.9* excluded); this cell is unstarred, i.e. the authors' own run. Qwen/Qwen3-8B (hybrid; thinking/non-thinking mode not printed, 'recommended evaluation configuration').",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "deepseek-r1-0528-qwen3-8b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.014,
    "source": "https://arxiv.org/html/2601.02346v1#S4.T5",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R report (TII, arXiv 2601.02346v1, CC BY 4.0), Table 5 'Code generation with reasoning evaluations', column 'TB Hard', row 'DeepSeek-R1-0528-Qwen3-8B' = 1.4. Sec. 4.1: 'Terminal-Bench (TB) Hard evaluates 47 hard tasks from the terminal-bench-core dataset (commit 74221fb) using the Terminus 2 agent. Each task is capped at 100 episodes and a 24-hour timeout, with model configurations set to the framework's default values'; 'For TB Hard ... we repeat each evaluation three times and report the average result'; 3 responses per query, max response length 32768. Baselines: 'we evaluated a range of SOTA models ... using each baseline's recommended evaluation configurations'; cells marked * are 'taken from Artificial Analysis' (Apriel-1.5-15b-Thinker 9.9* and GPT-OSS-20B 9.9* excluded); this cell is unstarred, i.e. the authors' own run. deepseek-ai/DeepSeek-R1-0528-Qwen3-8B (reasoning-only model).",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.028,
    "source": "https://arxiv.org/html/2601.02346v1#S4.T5",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Falcon-H1R report (TII, arXiv 2601.02346v1, CC BY 4.0), Table 5 'Code generation with reasoning evaluations', column 'TB Hard', row 'Qwen3-32B' = 2.8. Sec. 4.1: 'Terminal-Bench (TB) Hard evaluates 47 hard tasks from the terminal-bench-core dataset (commit 74221fb) using the Terminus 2 agent. Each task is capped at 100 episodes and a 24-hour timeout, with model configurations set to the framework's default values'; 'For TB Hard ... we repeat each evaluation three times and report the average result'; 3 responses per query, max response length 32768. Baselines: 'we evaluated a range of SOTA models ... using each baseline's recommended evaluation configurations'; cells marked * are 'taken from Artificial Analysis' (Apriel-1.5-15b-Thinker 9.9* and GPT-OSS-20B 9.9* excluded); this cell is unstarred, i.e. the authors' own run. Qwen/Qwen3-32B (hybrid; thinking/non-thinking mode not printed, 'recommended evaluation configuration').",
    "source_id": "report-2601.02346v1"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.036,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Terminal Bench Hard', column 'EXAONE 4.0 32B' = 3.6. Sec. 5.4: 'TerminalBench Hard, which corresponds to the 47 challenging tasks from the artificial analysis split'; App. C.2: 'all evaluated models utilized the same Terminus-2 agent implementation with reasoning enabled throughout the evaluation'; Sec. 5: 'we test all models in this report using Omni-Evaluator' (the authors' own runs). Columns: HyperCLOVA X 32B Think 9.9, EXAONE 4.0 32B 3.6, Qwen3-VL 32B-Thinking 6.9, InternVL3_5 38B-Thinking '–'. = LGAI-EXAONE/EXAONE-4.0-32B (cited as LG AI Research 2025), reasoning enabled.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.069,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Terminal Bench Hard', column 'Qwen3-VL 32B-Thinking' = 6.9. Sec. 5.4: 'TerminalBench Hard, which corresponds to the 47 challenging tasks from the artificial analysis split'; App. C.2: 'all evaluated models utilized the same Terminus-2 agent implementation with reasoning enabled throughout the evaluation'; Sec. 5: 'we test all models in this report using Omni-Evaluator' (the authors' own runs). Columns: HyperCLOVA X 32B Think 9.9, EXAONE 4.0 32B 3.6, Qwen3-VL 32B-Thinking 6.9, InternVL3_5 38B-Thinking '–'. = Qwen/Qwen3-VL-32B-Thinking (needs the proposed comparison-only model qwen3-vl-32b-thinking; without it this run has one catalogue model).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.0851,
    "source": "https://arxiv.org/html/2512.20848v1#S3.T3",
    "by": "lab",
    "date": "2025-12-23",
    "note": "NVIDIA, Nemotron 3 Nano technical report (arXiv 2512.20848v1, CC BY 4.0), Table 3, row 'Terminal Bench (hard subset)', column 'N-3-Nano' = 8.51 (model card bf77c31 prints 8.5). Sec. 3.4.1: Terminal Bench is 'not onboarded yet in our open source tools and for these we used their official open source implementation'. Own model only: the Qwen3-30B-A3B-Thinking-2507 (5.00) and GPT-OSS 20B (10.00) cells are excluded because the report says comparators use 'officially reported numbers whenever available; if a benchmark is not reported, we take the value from ArtificialAnalysis (AA)' and neither vendor reports TB-Hard (the card marks only its own measurements with *, and these cells are unstarred). Reasoning on (default). Routed to first-party-nvidia-scores via the '#S3.T3' fragment so NVIDIA's own TB-Hard rows form one vendor run with Nemotron 3 Super.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.2578,
    "source": "https://arxiv.org/html/2604.12374v1#S3.T5",
    "by": "lab",
    "date": "2026-04-14",
    "note": "NVIDIA, Nemotron 3 Super technical report (arXiv 2604.12374v1, CC BY 4.0), Table 5, row 'Terminal Bench (hard subset)', column 'N-3-Super' = 25.78 (card 2dc98e2 identical). Sec. 3.3: 'dedicated open-source packaged containers for ... Terminal Bench Hard (48 tasks)'; 'the same broad benchmark suite and evaluation stack as Nemotron 3 Nano'. BF16 checkpoint (Table 8 FP8 26.04 / NVFP4 24.48 not used). Own model only: Qwen3.5-122B-A10B 26.80 and GPT-OSS-120B 24.00 excluded because comparators are 'officially reported numbers whenever available; when not available ... sourcing values from reputable public aggregators ... or computing scores ourselves' without saying which. Routed to first-party-nvidia-scores via the '#S3.T5' fragment (one NVIDIA vendor run with Nemotron 3 Nano).",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "gemma-4-31b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.2708,
    "source": "https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4/blob/4135a98a9b728a548947683219633b25682223ac/README.md",
    "by": "independent",
    "date": "2026-07-13",
    "note": "NVIDIA ModelOpt, nvidia/Gemma-4-31B-IT-NVFP4 card @4135a98 (2026-07-13; repo licence Apache 2.0 per ai.google.dev/gemma/apache_2), 'Evaluation Results' table, row 'Terminal-Bench Hard (pass@1)', column 'Baseline (BF16)' = 27.08% (NVFP4 column 27.08% not used). Footnote: 'Baseline: Gemma-4-31B ... Benchmarked with temperature=1.0, top_p=0.95, max_new_tokens= 131072'; base_model google/gemma-4-31B-it. NVIDIA's own measurement of Google's model; thinking mode and agent harness not printed (NVIDIA's NeMo Evaluator 'terminal-bench-hard' = 47 tasks, Terminus-2).",
    "source_id": "nvidia-modelopt-card-baselines"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.5139,
    "source": "https://huggingface.co/nvidia/DeepSeek-V4-Pro-0813-NVFP4/blob/949138637e8e8fe190335be2396af62187dc8a83/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "NVIDIA ModelOpt, nvidia/DeepSeek-V4-Pro-0813-NVFP4 card @9491386 (2026-09-10; repo licence MIT), 'Evaluation' table, row 'MXFP4 (source)', column 'Terminal-Bench Hard' = 51.39 (NVFP4 row 50.69 not used). 'Baseline: DeepSeek-V4-Pro-0813, whose routed experts already ship in MXFP4' = the official deepseek-ai/DeepSeek-V4-Pro-0813 checkpoint (catalogue deepseek-v4-pro). 'All scores measured on SGLang at temperature 1.0, top_p 1.0, max reasoning effort ... Terminal-Bench Hard [is] single-run.' The -nvfp4-DSpark card @d9a7219 repeats the same measurement and is not used twice.",
    "source_id": "nvidia-modelopt-card-baselines"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.09,
    "source": "https://huggingface.co/MultiverseComputingCAI/Hypernova-60B-2602/blob/382391945c26d1ab65fc8bdc174865067cf80cd1/README.md",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Multiverse Computing, MultiverseComputingCAI/Hypernova-60B-2602 card @3823919 (2026-08-07; repo licence apache-2.0), 'Quantitative Results (Reported & Planned)' table, row 'Terminal Bench', column 'gpt-oss-20b' = 9. 'Reported numbers use the methodology described above'; the methodology's only Terminal-Bench entry is 'Terminal-Bench Hard (Artificial Analysis subset): Evaluation framework laude-institute/harbor == 0.1.43; Inference library vLLM == 0.15.0; Reasoning effort: high; Decoding: temperature = 1.0, top_p = 1.0, max-model-len = 131072; Agent: terminus-2, max episodes 100; repeats 3', so the row is TB-Hard. Multiverse's own run of OpenAI's open-weights model (the card's HyperNova 60B 2602 = 16 is a compressed gpt-oss-120b, not catalogued). = openai/gpt-oss-20b, reasoning effort high. Integer-rounded as printed.",
    "source_id": "multiverse-hypernova-60b-2602-card"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.22,
    "source": "https://huggingface.co/MultiverseComputingCAI/Hypernova-60B-2602/blob/382391945c26d1ab65fc8bdc174865067cf80cd1/README.md",
    "by": "independent",
    "date": "2026-08-07",
    "note": "Multiverse Computing, MultiverseComputingCAI/Hypernova-60B-2602 card @3823919 (2026-08-07; repo licence apache-2.0), 'Quantitative Results (Reported & Planned)' table, row 'Terminal Bench', column 'gpt-oss-120b' = 22. 'Reported numbers use the methodology described above'; the methodology's only Terminal-Bench entry is 'Terminal-Bench Hard (Artificial Analysis subset): Evaluation framework laude-institute/harbor == 0.1.43; Inference library vLLM == 0.15.0; Reasoning effort: high; Decoding: temperature = 1.0, top_p = 1.0, max-model-len = 131072; Agent: terminus-2, max episodes 100; repeats 3', so the row is TB-Hard. Multiverse's own run of OpenAI's open-weights model (the card's HyperNova 60B 2602 = 16 is a compressed gpt-oss-120b, not catalogued). = openai/gpt-oss-120b, reasoning effort high. Integer-rounded as printed. The later HyperNova 60B 2605 card @373c5c7 re-measures GPT-OSS-120B at 24.2 under the same harness (vLLM 0.13.0); not recorded (would be a single-catalogue-model run unless pooled).",
    "source_id": "multiverse-hypernova-60b-2602-card"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "tau2-telecom",
    "score": 0.9825,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/submission.json",
    "by": "lab",
    "date": "2026-01-26",
    "note": "Qwen's own submission (submitting_organization 'Qwen', submission_date 2026-01-26), model_name 'Qwen3-Max-Thinking'; results.telecom.pass_1 = 98.24561403508771 (rounded to 4 dp; = 448/456 = 114 tasks x 4 trials). methodology: tau2_bench_version v0.1.3, user_simulator gpt-4.1-2025-04-14, modified_prompts false, omitted_questions false, 'Evaluation conducted using standard tau2-bench protocol'. Checkpoint: the trajectory file committed with the submission at 70e700c (web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/trajectories/telecom_llm_agent_qwen3-max-2026-01-23_user_simulator_gpt-4.1-2025-04-14.json, removed from git in 20062c7) records agent llm 'qwen3-max-2026-01-23', temperature 0.0, 4 trials, max_steps 200; re-computed pass rate 0.98246. Thinking flag not recorded in llm_args (the snapshot is hybrid). Needs the proposed comparison-only model qwen3-max-2026-01-23.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "tau2-telecom",
    "score": 0.842,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2024_09_23/submission.json",
    "by": "lab",
    "date": "2025-09-23",
    "note": "Submission dir qwen3-max_qwen_2024_09_23 (submitting_organization 'Alibaba', model_name 'Qwen3-Max', model_release.release_date 2025-09-23, reference 'Qwen3-Max: Just Scale it' blog); results.telecom.pass_1 = 84.2. methodology: tau2_bench_version 0.1.3, user_simulator null, verification 'Unverified submission - no trajectory data available, unknown evaluation methodology'; notes 'Only average results are in the blog post. Evaluation limited to Pass^1, Pass^2 and Pass^4 scores only'. The submission's own submission_date/evaluation_date say 2024-09-23 (a typo for 2025: the model was released 2025-09-23); date set to the release date. Checkpoint: Qwen3-Max (Instruct) official release of 2025-09-23 = dated API snapshot qwen3-max-2025-09-23 (non-thinking). CAUTION: committed by Sierra staff (957599b / 9df0df8) from Qwen material; where the per-domain values come from is not stated.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-telecom",
    "score": 0.385,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'DeepSeek V3.1' = 38.50. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. Card note: DeepSeek-V3.1 evaluated in non-thinking mode (only setting this source reports).",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.225,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'Qwen3 MoE-2507' = 22.50. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. Column is the 235B-total/22B-active Qwen3 MoE '2507' update run in non-thinking mode; the LongCat-Flash technical report (arXiv 2509.01322v1 Sec. 5, read for identity only) names it 'Qwen3-235B-A22B (2507 version)' and says closed models were evaluated 'through their official APIs', i.e. Qwen3-235B-A22B-Instruct-2507.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-telecom",
    "score": 0.675,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'Kimi-K2' = 67.50. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. 1043B total / 32B active; card published before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-telecom",
    "score": 0.352,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'GPT-4.1' = 35.20. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. Reasoning: n/a (non-reasoning model).",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau2-telecom",
    "score": 0.462,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'Claude4 Sonnet' = 46.20. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. Card note: non-thinking mode (only setting this source reports).",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "tau2-telecom",
    "score": 0.165,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom) (avg@4)' (mean pass rate over 4 trials), column 'Gemini2.5 Flash' = 16.50. No '*' (the card's mark for values 'sourced from other public reports'), so this is Meituan's own run. User simulator and tau2-bench version not printed. Card note: non-thinking mode (only setting this source reports); stable gemini-2.5-flash (GA June 2025, card dated Sept 2025).",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-telecom",
    "score": 0.237,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom (Mean@4)', column 'DeepSeek-V3.1-Thinking' = 23.7. Unstarred, so Meituan's own run (the starred GPT-5-Thinking 96.7* is excluded). User simulator / version not printed. DeepSeek V3.1 in thinking mode (highest setting this source reports).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.473,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom (Mean@4)', column 'Qwen3-235B-A22B-Thinking-2507' = 47.3. Unstarred, so Meituan's own run (the starred GPT-5-Thinking 96.7* is excluded). User simulator / version not printed. ",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "tau2-telecom",
    "score": 0.561,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom (Mean@4)', column 'GLM-4.5' = 56.1. Unstarred, so Meituan's own run (the starred GPT-5-Thinking 96.7* is excluded). User simulator / version not printed. 355B/32B column; mode not printed (thinking card).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "tau2-telecom",
    "score": 0.675,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom (Mean@4)', column 'OpenAI-o3' = 67.5. Unstarred, so Meituan's own run (the starred GPT-5-Thinking 96.7* is excluded). User simulator / version not printed. Reasoning effort not printed.",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "tau2-telecom",
    "score": 0.383,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom (Mean@4)', column 'Gemini2.5-Pro' = 38.3. Unstarred, so Meituan's own run (the starred GPT-5-Thinking 96.7* is excluded). User simulator / version not printed. Stable Gemini 2.5 Pro (card dated Sept 2025); thinking budget not printed.",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-telecom",
    "score": 0.658,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "lab",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'Kimi K2 Instruct' = 65.8. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. Same figure as Moonshot's τ²-bench leaderboard submission (tau2-bench-leaderboard-moonshot); recorded again only as this run's own-model anchor.",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "tau2-telecom",
    "score": 0.325,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'DeepSeek-V3-0324' = 32.5. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. ",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.221,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'Qwen3-235B-A22B (non-thinking)' = 22.1. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. Hybrid April-2025 Qwen3-235B-A22B with thinking off (only setting this source reports).",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau2-telecom",
    "score": 0.452,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'Claude Sonnet 4 (w/o extended thinking)' = 45.2. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. No extended thinking (only setting this source reports).",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "claude-opus-4-20250514",
    "bench": "tau2-telecom",
    "score": 0.57,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'Claude Opus 4 (w/o extended thinking)' = 57.0. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. No extended thinking (only setting this source reports).",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-telecom",
    "score": 0.386,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'GPT-4.1' = 38.6. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. ",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "tau2-telecom",
    "score": 0.169,
    "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/fd1984e2b7a3350dbf7305fe73a4ede25c14de50/README.md",
    "by": "independent",
    "date": "2026-04-23",
    "note": "Moonshot Kimi-K2-Instruct card (@fd1984e; table first published with the 2025-07-11 release), 'Instruction model evaluation results', row 'Tau2 telecom', metric Avg@4, column 'Gemini 2.5 Flash Preview (05-20)' = 16.9. Unstarred (the card's * marks values 'taken directly from the model's tech report or blog'), so Moonshot's own run; 8k output-token limit for all metrics. User simulator not printed. Thinking setting not printed. (The Qwen3-30B-A3B-Instruct-2507 card later reprints 16.9 for 'Gemini-2.5-Flash Non-Thinking'.)",
    "source_id": "card-moonshotai-kimi-k2-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "tau2-telecom",
    "score": 0.8772,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI LFM2.5-8B-A1B card (@5dd2260), section 'Math and agentic workflows', row 'Qwen3.5-4B', column 'Tau² Telecom' = 87.72. Unmarked cell in Liquid's own comparison table; value differs from the model maker's own published figure, so not copied from it. Same run as the existing lfm2-5-8b-a1b 0.8807 and lfm2-8b-a1b 0.136 rows of this card. User simulator / version / reasoning setting not printed. ",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.2193,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI LFM2.5-8B-A1B card (@5dd2260), section 'Math and agentic workflows', row 'Qwen3-30B-A3B-Thinking-2507', column 'Tau² Telecom' = 21.93. Unmarked cell in Liquid's own comparison table; value differs from the model maker's own published figure, so not copied from it. Same run as the existing lfm2-5-8b-a1b 0.8807 and lfm2-8b-a1b 0.136 rows of this card. User simulator / version / reasoning setting not printed. (Qwen's own card prints 26.3.)",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau2-telecom",
    "score": 0.2237,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI LFM2.5-8B-A1B card (@5dd2260), section 'Math and agentic workflows', row 'Gemma-4-E2B-IT', column 'Tau² Telecom' = 22.37. Unmarked cell in Liquid's own comparison table; value differs from the model maker's own published figure, so not copied from it. Same run as the existing lfm2-5-8b-a1b 0.8807 and lfm2-8b-a1b 0.136 rows of this card. User simulator / version / reasoning setting not printed. (Gemma 4 report prints 19.7.)",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau2-telecom",
    "score": 0.2675,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Liquid AI LFM2.5-8B-A1B card (@5dd2260), section 'Math and agentic workflows', row 'Gemma-4-E4B-IT', column 'Tau² Telecom' = 26.75. Unmarked cell in Liquid's own comparison table; value differs from the model maker's own published figure, so not copied from it. Same run as the existing lfm2-5-8b-a1b 0.8807 and lfm2-8b-a1b 0.136 rows of this card. User simulator / version / reasoning setting not printed. (Gemma 4 report prints 18.4.)",
    "source_id": "card-liquidai-lfm2-5-8b-a1b"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "tau2-telecom",
    "score": 0.577,
    "source": "https://arxiv.org/html/2601.07022v1",
    "by": "independent",
    "date": "2026-01-11",
    "note": "Upstage, Solar Open Technical Report v1 (CC BY 4.0), Table 6 'English Benchmarks', category Agent, row 'Tau 2 Telecom', column 'gpt-oss-120b (117B, high)' = 57.7. Section 7.1: agentic capabilities measured on 'Tau 2 (Airline/Telecom/Retail) (Barres et al., 2025)'; user simulator / trials not printed. The report does not say outright who ran the baselines, but the same tables include Upstage in-house test sets (KBankMMLU, Ko-AIME, Ko Arena Hard v2) for every baseline and these τ² values are not published by the models' makers, so they are Upstage's runs. Reasoning effort high (highest of the two printed; medium = 47.4 not used).",
    "source_id": "report-2601.07022v1"
  },
  {
    "model": "glm-4-5-air",
    "bench": "tau2-telecom",
    "score": 0.281,
    "source": "https://arxiv.org/html/2601.07022v1",
    "by": "independent",
    "date": "2026-01-11",
    "note": "Upstage, Solar Open Technical Report v1 (CC BY 4.0), Table 6 'English Benchmarks', category Agent, row 'Tau 2 Telecom', column 'GLM-4.5-Air (110B)' = 28.1. Section 7.1: agentic capabilities measured on 'Tau 2 (Airline/Telecom/Retail) (Barres et al., 2025)'; user simulator / trials not printed. The report does not say outright who ran the baselines, but the same tables include Upstage in-house test sets (KBankMMLU, Ko-AIME, Ko Arena Hard v2) for every baseline and these τ² values are not published by the models' makers, so they are Upstage's runs. Thinking mode not printed.",
    "source_id": "report-2601.07022v1"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-telecom",
    "score": 0.261,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER HyperCLOVA X 32B Think report v1 (CC BY 4.0), Table 5, row 'Tau 2 – Telecom', column 'EXAONE 4.0 32B' = 26.1. User simulator GPT-4.1 (printed in the Table 5 caption and Table 6 block), Appendix C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'; the report cites Barres et al. (2025) for the suite. Reasoning mode.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "gpt-5-1",
    "bench": "tau2-telecom",
    "score": 0.809,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER HyperCLOVA X 32B Think report v1 (CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Telecom', column 'GPT-5.1 (Medium)' = 80.9. User simulator GPT-4.1 (printed in the Table 5 caption and Table 6 block), Appendix C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'; the report cites Barres et al. (2025) for the suite. Reasoning effort medium (the only setting printed). The GPT-5.1-user block (81.2) is not used.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.205,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER HyperCLOVA X 32B Think report v1 (CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Telecom', column 'Qwen3 235B-A22B' = 20.5. User simulator GPT-4.1 (printed in the Table 5 caption and Table 6 block), Appendix C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'; the report cites Barres et al. (2025) for the suite. Hybrid April-2025 Qwen3-235B-A22B in reasoning mode (Appendix C.1). The GPT-5.1-user block (24.2) is not used.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "nemotron-3-nano-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.422,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md#reasoning-benchmark-evaluations",
    "by": "lab",
    "date": "2026-08-24",
    "note": "NVIDIA's own figure for its own model: Nemotron-3-Nano-30B-A3B card (@bf77c31), 'Reasoning Benchmark Evaluations' table, row 'TauBench V2 (Telecom)', column 'NVIDIA-Nemotron-3-Nano-30B-A3B-BF16' = 42.2. Run via NeMo Evaluator SDK with the packaged Tau-2 Bench container. Comparison columns NOT used: the card says '* denotes the accuracy numbers are measured by us' and the unstarred Qwen3-30B-A3B-Thinking-2507 τ² cells (58.0/58.8/26.3) equal Qwen's own card. The card's TauBench V2 (Average) 49.0 is already in data as tau2-bench (card-nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16, a one-model run). Fragment-scoped to first-party-nvidia-scores so NVIDIA's own models form one evaluator.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "tau2-telecom",
    "score": 0.6436,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-25",
    "note": "NVIDIA's own figure for its own model: Nemotron-3-Super-120B-A12B card (@2dc98e2), 'Benchmarks' table, 'TauBench V2' block, row 'Telecom', column 'Nemotron 3 Super' = 64.36. 'dedicated open-source packaged containers for Tau-2 Bench (default prompt)'. Comparison columns (Qwen3.5-122B-A10B 95.00, GPT-OSS-120B 66.00) NOT used: provenance unclear (mixed precision; other cells match makers' or third-party figures). The card's TauBench V2 Average 61.15 is already in data as tau2-bench (one-model card run).",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "tau2-telecom",
    "score": 0.33,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-03-20",
    "note": "NVIDIA's own figure: Nemotron-3-Nano-4B card (@dfaf35d), 'Evaluation Results' — second table ('We also evaluated our model in Reasoning-On mode'), row 'Tau2-Telecom' = 33 (Reasoning-on, the highest setting; Reasoning-off table gives 24.9, not used). NeMo-Skills harness.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "tau2-telecom",
    "score": 0.73,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-07-06",
    "note": "LG's own figure for its own model: EXAONE 4.5 33B card (@570aa4b), 'Evaluation Results' > 'Language-only Tasks' table, row 'τ2-Bench (Telecom)', column 'EXAONE 4.5 33B (Reasoning)' = 73.0. Reasoning mode. Other columns (GPT-5 mini 74.1, Qwen3-VL 235B Thinking 44.7, Qwen3.5 27B 99.3) are other labs' cells and not used (card licence is not open; rule b). The K-EXAONE column (73.5) is LG's but K-EXAONE is not catalogued.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-telecom",
    "score": 0.237,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure for its own earlier model: K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table (reasoning mode), row 'τ2-Bench (Telecom)', column 'EXAONE 4.0 (Reasoning)' = 23.7 (Architecture row 'Dense', Total Params '32B' = EXAONE 4.0 32B). The card says details are in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND, so not used as a licensed copy). K-EXAONE's own 73.5 is not recorded (not catalogued). GPT-OSS / Qwen3-Thinking-2507 / DeepSeek-V3.2 columns not used (other labs; the Qwen cell equals Qwen's own 45.6).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "trinity-large-thinking",
    "bench": "tau2-telecom",
    "score": 0.947,
    "source": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking/blob/dc6a99b6b74202880e31491a445a94884c97890c/README.md#benchmarks",
    "by": "lab",
    "date": "2026-05-28",
    "note": "Arcee's own figure: Trinity-Large-Thinking card (@dc6a99b), 'Benchmarks' table, row 'Tau2-Telecom', column 'Trinity-Large-Thinking' = 94.7. Harness/user simulator/version not printed. Opus-4.6, GLM-5, MiniMax-M2.7, Kimi-K2.5 columns not used (other labs; card licence OpenMDW-1.1 recorded under rule b). SINGLE-MODEL RUN for now (Trinity-Large-Preview's card has no telecom row); a later Arcee card would join it.",
    "source_id": "first-party-arcee-scores"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "tau3-telecom",
    "score": 0.914,
    "source": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/blob/22b2b868a15677cfa6061277ed2f653d1349a9ab/README.md#agentic-benchmarks",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Mistral's own figure: Mistral-Medium-3.5-128B card (@22b2b86), 'Benchmarks' > 'Agentic Benchmarks', sentence 'It scores 91.4% on τ³-Telecom and 77.6% on SWE-Bench Verified.' τ³ (tau2-bench >= 1.0) telecom, so tau3-telecom, NOT tau2-telecom. Version/user simulator/reasoning effort not printed (card recommends reasoning_effort='high'). SINGLE-MODEL RUN (no other Mistral τ³ figure).",
    "source_id": "first-party-mistral-scores"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.184,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own run in the Qwen/Qwen3-4B-Instruct-2507 card (@cdbee75), performance table row 'TAU2-Telecom', column 'Qwen3-30B-A3B Non-Thinking' = 18.4. Same-lab model (the ledger entry takes own-lab cells only). Joins the existing qwen3-4b-instruct-2507 0.132 row of this card so this card's tau2-telecom run counts (it is a one-model run today). Hybrid Qwen3-30B-A3B with thinking off: the only setting this card reports for it (instruct-model comparison). GPT-4.1-nano cell is '-'.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-telecom",
    "score": 0.175,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own run in the Qwen/Qwen3-4B-Instruct-2507 card (@cdbee75), performance table row 'TAU2-Telecom', column 'Qwen3-4B Non-Thinking' = 17.5. Same-lab model (the ledger entry takes own-lab cells only). Joins the existing qwen3-4b-instruct-2507 0.132 row of this card so this card's tau2-telecom run counts (it is a one-model run today). Hybrid Qwen3-4B with thinking off (only setting this card reports).",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "tau2-telecom",
    "score": 0.982,
    "source": "https://www-cdn.anthropic.com/bf10f64990cfda0ba858290be7b8cc6317685f47/Claude%20Opus%204.5%20System%20Card.pdf#page=26",
    "by": "lab",
    "date": "2025-11-24",
    "note": "Anthropic 'System Card: Claude Opus 4.5' (November 2025; changelog dated November 24, 2025), section 2.8 'τ2-bench', Table 2.8.A, row 'Telecom', column 'Claude Opus 4.5', printed '98.2%'. The same value appears in Table 2.3.A on PDF page 19 as 'τ²-Bench (Telecom) 98.2%' with footnote 4, 'Without extended thinking'. Settings: no extended thinking, the only setting the card reports for this cell. Table 2.3.A caption: 'average over 5 trials ... 200k context window, default effort (high), and default sampling settings'. Modified setup, quoted from the Table 2.8.A caption: 'All above results used Claude Opus 4.1 to simulate the user and included a prompt addendum instructing Claude to better target its known failure modes when using the vanilla prompt. A prompt addendum was also added to the Telecom User prompt to avoid failure modes from the user ending the interaction incorrectly.' Original τ²-bench Telecom domain; the card's separate 'Airline (corrected)' row is not relevant here. Vendor-reported (rule b), hand-recorded from a single fetch of the PDF; the page anchor is the physical PDF page.",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "tau2-telecom",
    "score": 0.715,
    "source": "https://www-cdn.anthropic.com/bf10f64990cfda0ba858290be7b8cc6317685f47/Claude%20Opus%204.5%20System%20Card.pdf#page=26",
    "by": "lab",
    "date": "2025-11-24",
    "note": "Anthropic 'System Card: Claude Opus 4.5' (November 2025), section 2.8, Table 2.8.A, row 'Telecom', column 'Claude Opus 4.1', printed '71.5%'. Table 2.3.A on PDF page 19 also prints 71.5%, with no footnote. Settings for the Opus 4.1 column are not restated; the Table 2.3.A caption says results are 'an average over 5 trials and run with a 64k thinking budget ... Exceptions noted in footnotes'. Modified setup, quoted from the Table 2.8.A caption: 'All above results used Claude Opus 4.1 to simulate the user and included a prompt addendum instructing Claude to better target its known failure modes when using the vanilla prompt. A prompt addendum was also added to the Telecom User prompt to avoid failure modes from the user ending the interaction incorrectly.' Anthropic's own model (Claude Opus 4.1 has one snapshot, claude-opus-4-1-20250805). This is not a duplicate: Anthropic's τ²-leaderboard submission for Opus 4.1 (tau2-bench-leaderboard-anthropic) has only original τ-bench retail and airline. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "tau2-telecom",
    "score": 0.9925,
    "source": "https://www-cdn.anthropic.com/6a5fa276ac68b9aeb0c8b6af5fa36326e0e166dd/Claude%20Opus%204.6%20System%20Card.pdf#page=21",
    "by": "lab",
    "date": "2026-02-05",
    "note": "Anthropic 'System Card: Claude Opus 4.6' (February 2026; announcement dated 2026-02-05), section 2.7 'τ2-bench', prose: 'Claude Opus 4.6 achieves a score of 99.25% (Telecom) and 91.89% (Retail), averaged over 5 trials, each run with adaptive thinking, max effort, and default sampling settings (temperature, top_p).' The rounded 99.3% is in Table 2.3.A on PDF page 18. Settings: adaptive thinking, max effort (the highest). This card states no user simulator or prompt addendum; the Opus 4.5 card's setup used Claude Opus 4.1 as the user plus agent and Telecom-user prompt addenda, but this card does not say the setup was reused. Airline is not reported ('policy loopholes ... have not yet been incorporated upstream'). Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "tau2-telecom",
    "score": 0.979,
    "source": "https://www-cdn.anthropic.com/bbd8ef16d70b7a1665f14f306ee88b53f686aa75/Claude%20Sonnet%204.6%20System%20Card.pdf#page=18",
    "by": "lab",
    "date": "2026-02-17",
    "note": "Anthropic 'System Card: Claude Sonnet 4.6' (February 17, 2026), section 2.5 'τ2-bench', prose: 'Claude Sonnet 4.6 achieved a score of 97.9% (Telecom) and 91.7% (Retail), averaged over 10 trials, each run with adaptive thinking, max effort, and default sampling settings (temperature, top_p).' Table 2.1.A on PDF page 15 also prints 'τ²-bench ... Telecom 97.9%'. Settings: adaptive thinking, max effort. The card states no user simulator or prompt addendum. Airline is not reported. Vendor-reported (rule b).",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "tau2-telecom",
    "score": 0.98,
    "source": "https://storage.googleapis.com/deepmind-media/gemini/gemini_3_pro_model_evaluation.pdf#page=2",
    "by": "lab",
    "date": "2025-11-18",
    "note": "Google DeepMind 'Model Evaluation – Approach, Methodology & Results: Gemini 3 Pro' (the PDF that deepmind.google/models/evals-methodology/gemini-3-pro/ redirects to; HTTP Last-Modified 18 Nov 2025; 'as of November, 2025'), 'Tool Use' bullet, printed: 'All scores reported above are the average of scores on the three individual categories: Retail, Airline and Telecom. Gemini 3.0 Pro scores 85.3%, 73.0% and 98.0% on these categories respectively.' Telecom = 98.0%. The aggregate τ²-bench is printed only in the PDF's results table, which is an image; it is not recorded. Modified setup, quoted: 'τ2-bench results for Gemini use standard sierra framework with a prompt adjustment to provide instructions relevant to each environment. The user model uses Gemini with a system instruction.' Checkpoint: 'run with the Gemini API for the model-id gemini-3-pro-preview with default sampling settings'. The PDF does not state the thinking level; the Gemini 3.1 Pro model card prints the same 98.0% under 'Gemini 3 Pro Thinking (High)'. Google's own run of its own model. Vendor-reported (rule b).",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "tau2-telecom",
    "score": 0.993,
    "source": "https://deepmind.google/models/model-cards/gemini-3-1-pro",
    "by": "lab",
    "date": "2026-02-19",
    "note": "Google DeepMind Gemini 3.1 Pro model card, HTML page ('Published 19 February 2026'), Evaluation > Results table ('Results as of February 2026'), row 'τ2-bench / Agentic and tool use / Telecom', column 'Gemini 3.1 Pro Thinking (High)', printed '99.3%'. Retail (90.8%) is also printed; there is no aggregate. Settings: thinking level High. Methodology PDF (deepmind.google/models/evals-methodology/gemini-3-1-pro, 'Published: February, 2026'): 'run with the Gemini API for the model-id gemini-3.1-pro-preview with default sampling settings'. Modified setup, quoted: 'τ2-bench results for Gemini use standard sierra framework with a prompt adjustment to provide instructions relevant to each environment. The user model uses Gemini with a system instruction. The airline version was excluded due to lower quality grading.' Only the own-model column is used; the Sonnet 4.6, Opus 4.6 and GPT-5.2 columns are excluded. Vendor-reported (rule b). URL given without its trailing slash; the ledger entry is exact:false, so it matches either way.",
    "source_id": "first-party-google-scores"
  },
  {
    "model": "gpt-5",
    "bench": "tau2-telecom",
    "score": 0.967,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-5 (high)', printed '96.7%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Settings: reasoning effort high. The page body repeats the figure: 'achieving SOTA results on τ2-bench telecom (96.7%)' and 'tool calling (96.7% on τ2-bench telecom)'. The GPT-5.1 developer post repeats 96.7% for 'GPT‑5 (high)', the same figure, so it is not added again. The existing Sierra-run row for GPT-5 (0.9583, tau2-bench-leaderboard-sierra) is a separate measurement.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-telecom",
    "score": 0.741,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-5 mini (high)', printed '74.1%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Settings: reasoning effort high. The same 74.1% for GPT-5 mini is printed in the GPT-5.4 mini/nano post, which the 2026-10-07 pass left in 'unmatched'; it appears to be this same August 2025 figure.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-nano",
    "bench": "tau2-telecom",
    "score": 0.355,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-5 nano (high)', printed '35.5%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Settings: reasoning effort high.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o3",
    "bench": "tau2-telecom",
    "score": 0.582,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'OpenAI o3 (high)', printed '58.2%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Settings: reasoning effort high. The Introducing GPT-5 post also carries 0.582 for o3, but only in a Vega-Lite chart spec, which is not used.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-telecom",
    "score": 0.405,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'OpenAI o4-mini (high)', printed '40.5%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Settings: reasoning effort high. This is a separate measurement from the existing Sierra-run row (0.502, original τ²-bench paper evaluation).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-telecom",
    "score": 0.34,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-4.1', printed '34.0%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Non-reasoning model. Caution: the value equals the existing Sierra-run row (0.34, tau2-bench-leaderboard-sierra, from the τ²-bench paper). OpenAI credits no one else and gives the same harness note, but this could be the paper's figure.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "tau2-telecom",
    "score": 0.44,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-4.1 mini', printed '44.0%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Non-reasoning model. This differs from the existing Sierra-run row (0.489), so it is a separate measurement.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "tau2-telecom",
    "score": 0.121,
    "source": "https://openai.com/index/introducing-gpt-5-for-developers",
    "by": "lab",
    "date": "2025-08-07",
    "note": "OpenAI 'Introducing GPT‑5 for developers' (August 7, 2025), Appendix, 'Function Calling' table, row 'Tau2-bench telecom', column 'GPT-4.1 nano', printed '12.1%' (printed as 'Tau 2 -bench telecom'). Read from Internet Archive capture 20261006155708 of https://openai.com/index/introducing-gpt-5-for-developers/ (openai.com returns HTTP 403 to direct fetches). Page text: 'In τ2-bench, a model must use tools to accomplish a customer service task, where there may be a user who can communicate and can take actions on the world state. Reasoning models were run with high reasoning effort.' The page states no prompt change and no user model; the GPT-5.1 and GPT-5.2 posts' prompt additions are not mentioned here. Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. Non-reasoning model. GPT-4.1 nano has a single snapshot, gpt-4.1-nano-2025-04-14.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "tau2-telecom",
    "score": 0.962,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/assets/paper.pdf",
    "by": "lab",
    "date": "2025-12-01",
    "note": "DeepSeek-V3.2 tech report PDF committed at assets/paper.pdf in deepseek-ai/DeepSeek-V3.2 @a7e62ac (repo licence MIT: README front-matter 'license: mit', root LICENSE 'MIT License'); linked as 'Tech report' from api-docs.deepseek.com/news/news251201. Section 4 text: 'For τ2-bench, we employ the model itself as the user agent, achieving final category scores of 63.8 (Airline), 81.1 (Retail), and 96.2 (Telecom).' Printed 96.2 -> 0.962. Thinking mode ('Tool-use benchmarks are evaluated using the standard function call format, wherein models are configured to thinking mode'). User simulator = DeepSeek-V3.2 itself, not Sierra's default. An aggregate τ²-Bench (Pass@1) 80.3 is also printed (Table 2; non-thinking 77.2 in the appendix table) - no non-thinking Telecom split. DeepSeek-V3.2-Speciale has no τ²-bench result (API had no tool calls). Text identical to arXiv 2512.02556v1. Alternative copy with the same sentence: https://arxiv.org/html/2512.02556v1 (arXiv non-exclusive licence; would need first-party-deepseek-scores add_match) - use one source only.",
    "source_id": "card2-deepseek-ai-deepseek-v3-2"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "tau2-telecom",
    "score": 0.241,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'GPT-4o-2024-11-20' = 24.1. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. Dated snapshot printed.",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-telecom",
    "score": 0.389,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'GPT-4.1' = 38.9. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. (Sierra's own run: 34.0; Moonshot's: 38.6.)",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "tau2-telecom",
    "score": 0.329,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'DeepSeek-V3-0324' = 32.9. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. (Moonshot's run: 32.5; Qwen card: 34.0.)",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "qwen3-8b",
    "bench": "tau2-telecom",
    "score": 0.191,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'Qwen3-8B Non-thinking' = 19.1. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. Hybrid Qwen3-8B with thinking off (the paper runs every model non-thinking); MUA-RL base model.",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "qwen3-14b",
    "bench": "tau2-telecom",
    "score": 0.299,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'Qwen3-14B Non-thinking' = 29.9. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. Hybrid Qwen3-14B, thinking off; MUA-RL base model.",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-telecom",
    "score": 0.248,
    "source": "https://arxiv.org/html/2508.18669v1",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Meituan MUA-RL paper v1 (CC BY 4.0), Table 1 'Performance comparison of different models on TAU-Bench. Each model runs in non-thinking mode.', column TAU2 'Telecom', row 'Qwen3-32B Non-thinking' = 24.8. Sec. 4.1 Evaluation: 'For TAU1-Bench and TAU2-Bench, we use GPT-4.1 as the user simulator. For each test set, we conduct four repeated tests and take the average.' Meituan authors, so independent for this model. Hybrid Qwen3-32B, thinking off; MUA-RL base model.",
    "source_id": "paper-2508.18669v1-mua-rl"
  },
  {
    "model": "gpt-5-mini",
    "bench": "tau2-telecom",
    "score": 0.713,
    "source": "https://arxiv.org/html/2606.28715v2",
    "by": "independent",
    "date": "2026-09-05",
    "note": "SEATauBench v2 (CC BY 4.0), Appendix G Table 17 'English Baseline (S1) results', Domain Telecom, row 'GPT-5-mini', column 'EN p@1' = 0.713. Sec. 4: telecom = original τ²-Bench tasks (114); user LLM Qwen3-235B-A22B-Instruct (not the default GPT-4.1), GPT-4.1 NL-assertion judge, three trials per task (q=3), each provider's default hyperparameters, no model-specific tuning. Provider default reasoning effort (not the highest; the only setting reported).",
    "source_id": "paper-2606.28715v2-seataubench"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.357,
    "source": "https://arxiv.org/html/2606.28715v2",
    "by": "independent",
    "date": "2026-09-05",
    "note": "SEATauBench v2 (CC BY 4.0), Appendix G Table 17 'English Baseline (S1) results', Domain Telecom, row 'Qwen3-235B', column 'EN p@1' = 0.357. Sec. 4: telecom = original τ²-Bench tasks (114); user LLM Qwen3-235B-A22B-Instruct (not the default GPT-4.1), GPT-4.1 NL-assertion judge, three trials per task (q=3), each provider's default hyperparameters, no model-specific tuning. Caption: 'Qwen3-235B refers to Qwen3-235B-A22B-Instruct-2507.'",
    "source_id": "paper-2606.28715v2-seataubench"
  },
  {
    "model": "kimi-k2-5",
    "bench": "tau2-telecom",
    "score": 0.997,
    "source": "https://arxiv.org/html/2606.28715v2",
    "by": "independent",
    "date": "2026-09-05",
    "note": "SEATauBench v2 (CC BY 4.0), Appendix G Table 17 'English Baseline (S1) results', Domain Telecom, row 'Kimi-K2.5', column 'EN p@1' = 0.997. Sec. 4: telecom = original τ²-Bench tasks (114); user LLM Qwen3-235B-A22B-Instruct (not the default GPT-4.1), GPT-4.1 NL-assertion judge, three trials per task (q=3), each provider's default hyperparameters, no model-specific tuning. Provider default settings (thinking mode not stated).",
    "source_id": "paper-2606.28715v2-seataubench"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "tau2-telecom",
    "score": 0.132,
    "source": "https://arxiv.org/html/2606.03892v2",
    "by": "independent",
    "date": "2026-06-03",
    "note": "IBM Research 'Synthesize and Reward' v2 (CC BY 4.0), Table 3, row 'Qwen3-4B / Base', τ²-bench column 'Telecom' = 13.2. Sec. 4.1: user simulator Qwen3-VL-235B-A22B-Instruct 'across all runs to keep evaluations directly comparable' (not the default GPT-4.1). Checkpoint from the paper's footnoted HF link: Qwen/Qwen3-4B-Instruct-2507. (Equals Qwen's own card figure 13.2 = 15/114; may be coincidence.)",
    "source_id": "paper-2606.03892v2-synthesize-and-reward"
  },
  {
    "model": "qwen3-8b",
    "bench": "tau2-telecom",
    "score": 0.246,
    "source": "https://arxiv.org/html/2606.03892v2",
    "by": "independent",
    "date": "2026-06-03",
    "note": "IBM Research 'Synthesize and Reward' v2 (CC BY 4.0), Table 3, row 'Qwen3-8B / Base', τ²-bench column 'Telecom' = 24.6. Sec. 4.1: user simulator Qwen3-VL-235B-A22B-Instruct 'across all runs to keep evaluations directly comparable' (not the default GPT-4.1). Checkpoint from the paper's footnoted HF link: Qwen/Qwen3-8B. Thinking mode not stated.",
    "source_id": "paper-2606.03892v2-synthesize-and-reward"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "tau2-telecom",
    "score": 0.167,
    "source": "https://arxiv.org/html/2606.03892v2",
    "by": "independent",
    "date": "2026-06-03",
    "note": "IBM Research 'Synthesize and Reward' v2 (CC BY 4.0), Table 3, row 'Qwen2.5-7B / Base', τ²-bench column 'Telecom' = 16.7. Sec. 4.1: user simulator Qwen3-VL-235B-A22B-Instruct 'across all runs to keep evaluations directly comparable' (not the default GPT-4.1). Checkpoint from the paper's footnoted HF link: Qwen/Qwen2.5-7B-Instruct. ",
    "source_id": "paper-2606.03892v2-synthesize-and-reward"
  },
  {
    "model": "granite-4-1-8b",
    "bench": "tau2-telecom",
    "score": 0.175,
    "source": "https://arxiv.org/html/2606.03892v2",
    "by": "lab",
    "date": "2026-06-03",
    "note": "IBM Research 'Synthesize and Reward' v2 (CC BY 4.0), Table 3, row 'Granite-4.1-8B / Base', τ²-bench column 'Telecom' = 17.5. Sec. 4.1: user simulator Qwen3-VL-235B-A22B-Instruct 'across all runs to keep evaluations directly comparable' (not the default GPT-4.1). Checkpoint from the paper's footnoted HF link: ibm-granite/granite-4.1-8b-instruct. IBM authors, IBM model: by 'lab'.",
    "source_id": "paper-2606.03892v2-synthesize-and-reward"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "tau2-telecom",
    "score": 0.088,
    "source": "https://arxiv.org/html/2602.22697v1",
    "by": "independent",
    "date": "2026-02-26",
    "note": "Meituan InteractCS-RL v1 (CC BY 4.0), Table 2 'Performance of our InteractCS-RL in τ²-Bench', row 'Qwen2.5-7B', Telecom 'Pass@1' = 8.8%. Sec. 5.3: agents 'based on Qwen2.5-7/14B-Instruct'. τ²-bench setup in App. A.2 (user simulator for τ² not separately stated; the paper's own benchmark uses DeepSeek-V3.2 as user). Values are k/114 (single pass over the 114 tasks). Checkpoint: Qwen2.5-7B-Instruct.",
    "source_id": "paper-2602.22697v1-interactcs-rl"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "tau2-telecom",
    "score": 0.175,
    "source": "https://arxiv.org/html/2602.22697v1",
    "by": "independent",
    "date": "2026-02-26",
    "note": "Meituan InteractCS-RL v1 (CC BY 4.0), Table 2 'Performance of our InteractCS-RL in τ²-Bench', row 'Qwen2.5-14B', Telecom 'Pass@1' = 17.5%. Sec. 5.3: agents 'based on Qwen2.5-7/14B-Instruct'. τ²-bench setup in App. A.2 (user simulator for τ² not separately stated; the paper's own benchmark uses DeepSeek-V3.2 as user). Values are k/114 (single pass over the 114 tasks). Checkpoint: Qwen2.5-14B-Instruct.",
    "source_id": "paper-2602.22697v1-interactcs-rl"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "mmlu-pro",
    "score": 0.857,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MMLU-Pro', column 'Qwen3-Max-Thinking' = 85.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "mmlu-redux",
    "score": 0.928,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MMLU-Redux', column 'Qwen3-Max-Thinking' = 92.8 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "supergpqa",
    "score": 0.673,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'SuperGPQA', column 'Qwen3-Max-Thinking' = 67.3 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "c-eval",
    "score": 0.937,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'C-Eval', column 'Qwen3-Max-Thinking' = 93.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "ifeval-unspecified",
    "score": 0.934,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'IFEval', column 'Qwen3-Max-Thinking' = 93.4 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "ifbench-unspecified",
    "score": 0.709,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'IFBench', column 'Qwen3-Max-Thinking' = 70.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "multichallenge",
    "score": 0.633,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MultiChallenge', column 'Qwen3-Max-Thinking' = 63.3 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "longbench-v2",
    "score": 0.606,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'LongBench v2', column 'Qwen3-Max-Thinking' = 60.6 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "gpqa-unspecified",
    "score": 0.874,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'GPQA', column 'Qwen3-Max-Thinking' = 87.4 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "hle",
    "score": 0.302,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'HLE', column 'Qwen3-Max-Thinking' = 30.2 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "livecodebench-v6",
    "score": 0.859,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'LiveCodeBench v6', column 'Qwen3-Max-Thinking' = 85.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "hmmt-feb-2025",
    "score": 0.98,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'HMMT Feb 25', column 'Qwen3-Max-Thinking' = 98.0 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "hmmt-nov-2025",
    "score": 0.947,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'HMMT Nov 25', column 'Qwen3-Max-Thinking' = 94.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "imo-answerbench",
    "score": 0.839,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'IMOAnswerBench', column 'Qwen3-Max-Thinking' = 83.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "aime-2026",
    "score": 0.933,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'AIME26', column 'Qwen3-Max-Thinking' = 93.3 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "bfcl-v4",
    "score": 0.677,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'BFCL-V4', column 'Qwen3-Max-Thinking' = 67.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "tau2-bench",
    "score": 0.846,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'TAU2-Bench', column 'Qwen3-Max-Thinking' = 84.6 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "vita-bench",
    "score": 0.409,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'VITA-Bench', column 'Qwen3-Max-Thinking' = 40.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "deepplanning",
    "score": 0.287,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'DeepPlanning', column 'Qwen3-Max-Thinking' = 28.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "toolathlon",
    "score": 0.188,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'Tool Decathlon', column 'Qwen3-Max-Thinking' = 18.8 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "mcpmark",
    "score": 0.335,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MCP-Mark', column 'Qwen3-Max-Thinking' = 33.5 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "hle-with-tools",
    "score": 0.498,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'HLE w/ tool', column 'Qwen3-Max-Thinking' = 49.8 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "browsecomp",
    "score": 0.539,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'BrowseComp', column 'Qwen3-Max-Thinking' = 53.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "browsecomp-zh",
    "score": 0.609,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'BrowseComp-zh', column 'Qwen3-Max-Thinking' = 60.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "widesearch",
    "score": 0.579,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'WideSearch', column 'Qwen3-Max-Thinking' = 57.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "seal-0",
    "score": 0.469,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'Seal-0', column 'Qwen3-Max-Thinking' = 46.9 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "mmmlu",
    "score": 0.844,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MMMLU', column 'Qwen3-Max-Thinking' = 84.4 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "mmlu-prox",
    "score": 0.785,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MMLU-ProX', column 'Qwen3-Max-Thinking' = 78.5 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "nova-63",
    "score": 0.542,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'NOVA-63', column 'Qwen3-Max-Thinking' = 54.2 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "include",
    "score": 0.823,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'INCLUDE', column 'Qwen3-Max-Thinking' = 82.3 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "global-piqa",
    "score": 0.86,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'Global PIQA', column 'Qwen3-Max-Thinking' = 86.0 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "polymath",
    "score": 0.647,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'PolyMATH', column 'Qwen3-Max-Thinking' = 64.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "wmt24pp",
    "score": 0.776,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'WMT24++', column 'Qwen3-Max-Thinking' = 77.6 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "maxife",
    "score": 0.84,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'MAXIFE', column 'Qwen3-Max-Thinking' = 84.0 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "swe-bench-verified",
    "score": 0.753,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'SWE-bench Verified', column 'Qwen3-Max-Thinking' = 75.3 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "swe-bench-multilingual",
    "score": 0.667,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'SWE-bench Multilingual', column 'Qwen3-Max-Thinking' = 66.7 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "seccodebench",
    "score": 0.575,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'SecCodeBench', column 'Qwen3-Max-Thinking' = 57.5 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "terminal-bench-2",
    "score": 0.225,
    "note": "Qwen3.5-397B-A17B model card @8472618 (Apache-2.0), Language table, row 'Terminal Bench 2', column 'Qwen3-Max-Thinking' = 22.5 Column 'Qwen3-Max-Thinking' mapped 2026-10-09 to the dated snapshot qwen3-max-2026-01-23 (Alibaba's release of Qwen3-Max-Thinking, 23 Jan 2026), the only Qwen3-Max-Thinking release before this card (Feb 2026); thinking mode. Same-lab column (by: lab).",
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "mistral-large-3",
    "bench": "wmt26-gemba-esa",
    "score": 0.816,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Large 3', column 'GEMBA WMT26' = 81.6. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "mistral-large-3",
    "bench": "wmt24pp",
    "score": 0.783,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Large 3', column 'xCOMETxl WMT24++' = 78.3. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "mistral-large-3",
    "bench": "wmt25-terminology",
    "score": 0.912,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Large 3', column 'Terminology' = 91.2. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "wmt26-gemba-esa",
    "score": 0.816,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Qwen 3.5 397B', column 'GEMBA WMT26' = 81.6. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "wmt24pp",
    "score": 0.785,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Qwen 3.5 397B', column 'xCOMETxl WMT24++' = 78.5. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "wmt25-terminology",
    "score": 0.87,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Qwen 3.5 397B', column 'Terminology' = 87.0. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gemma-4-31b",
    "bench": "wmt26-gemba-esa",
    "score": 0.795,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Gemma 4 31b', column 'GEMBA WMT26' = 79.5. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gemma-4-31b",
    "bench": "wmt24pp",
    "score": 0.792,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Gemma 4 31b', column 'xCOMETxl WMT24++' = 79.2. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gemma-4-31b",
    "bench": "wmt25-terminology",
    "score": 0.921,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Gemma 4 31b', column 'Terminology' = 92.1. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "inkling-small",
    "bench": "wmt26-gemba-esa",
    "score": 0.785,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Inkling Small', column 'GEMBA WMT26' = 78.5. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "inkling-small",
    "bench": "wmt24pp",
    "score": 0.787,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Inkling Small', column 'xCOMETxl WMT24++' = 78.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "inkling-small",
    "bench": "wmt25-terminology",
    "score": 0.911,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Inkling Small', column 'Terminology' = 91.1. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "wmt26-gemba-esa",
    "score": 0.782,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Muse Glimmer 30B', column 'GEMBA WMT26' = 78.2. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "wmt24pp",
    "score": 0.777,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Muse Glimmer 30B', column 'xCOMETxl WMT24++' = 77.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "wmt25-terminology",
    "score": 0.901,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Muse Glimmer 30B', column 'Terminology' = 90.1. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-plus",
    "bench": "wmt26-gemba-esa",
    "score": 0.765,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A+', column 'GEMBA WMT26' = 76.5. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-plus",
    "bench": "wmt24pp",
    "score": 0.81,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A+', column 'xCOMETxl WMT24++' = 81.0. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-plus",
    "bench": "wmt25-terminology",
    "score": 0.687,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A+', column 'Terminology' = 68.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "glm-5-2",
    "bench": "wmt26-gemba-esa",
    "score": 0.765,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GLM 5.2', column 'GEMBA WMT26' = 76.5. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "glm-5-2",
    "bench": "wmt24pp",
    "score": 0.788,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GLM 5.2', column 'xCOMETxl WMT24++' = 78.8. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "glm-5-2",
    "bench": "wmt25-terminology",
    "score": 0.782,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GLM 5.2', column 'Terminology' = 78.2. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "wmt26-gemba-esa",
    "score": 0.757,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Medium 3.5', column 'GEMBA WMT26' = 75.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "wmt24pp",
    "score": 0.767,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Medium 3.5', column 'xCOMETxl WMT24++' = 76.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "wmt25-terminology",
    "score": 0.903,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Mistral Medium 3.5', column 'Terminology' = 90.3. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "wmt26-gemba-esa",
    "score": 0.723,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GPT-OSS 120B', column 'GEMBA WMT26' = 72.3. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "wmt24pp",
    "score": 0.766,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GPT-OSS 120B', column 'xCOMETxl WMT24++' = 76.6. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "wmt25-terminology",
    "score": 0.838,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'GPT-OSS 120B', column 'Terminology' = 83.8. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "wmt26-gemba-esa",
    "score": 0.671,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Nemotron 3 Ultra 550B', column 'GEMBA WMT26' = 67.1. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "wmt25-terminology",
    "score": 0.848,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Nemotron 3 Ultra 550B', column 'Terminology' = 84.8. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "wmt26-gemba-esa",
    "score": 0.619,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Qwen 3.6 27B', column 'GEMBA WMT26' = 61.9. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "wmt25-terminology",
    "score": 0.892,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "independent",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Qwen 3.6 27B', column 'Terminology' = 89.2. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (the authors' own runs). Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "nano-banana-2-1",
    "bench": "lmarena-image",
    "score": 1327.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_to_image/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-07",
    "note": "text_to_image, category overall, split latest; exact entry 'gemini-nano-banana-2.1'; 5,310 votes; published 95% interval 1319.0–1336.5; leaderboard_publish_date 2026-10-07. Pinned to dataset revision 088c6079 (file identical to the 1361ea04 2026-10-07 update commit).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nano-banana-2-1",
    "bench": "lmarena-image-edit",
    "score": 1428.1,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'gemini-nano-banana-2.1'; 12,985 votes; published 95% interval 1422.3–1433.9; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (file identical to the 9490abd9 2026-10-06 update commit).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "chatgpt-image-latest",
    "bench": "lmarena-image-edit",
    "score": 1388.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'chatgpt-image-latest-high-fidelity (20251216)'; 629,464 votes; published 95% interval 1385.9–1391.3; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (file identical to the 9490abd9 2026-10-06 update commit).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "omnigen-v1",
    "bench": "omnicontext-average",
    "score": 0.434,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "lab",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'OmniGen [98]' = 4.34 (cited as Xiao et al. 2025, 'Omnigen: unified image generation', i.e. OmniGen v1). OmniContext is introduced in this report (Section 5.4; GPT-4.1 judge), so every cell is the authors' own run; OmniGen v1 is BAAI's own earlier model, hence by lab. 0-10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "uno",
    "bench": "omnicontext-average",
    "score": 0.471,
    "source": "https://arxiv.org/html/2506.18871v4",
    "by": "independent",
    "date": "2026-04-21",
    "note": "OmniGen2 report (BAAI), arXiv:2506.18871v4, CC BY 4.0, Table 4 'Overall comparison of existing models on our proposed OmniContext benchmark', column 'Average', row 'UNO [97]' = 4.71 (cited as S. Wu et al. 2025, 'Less-to-more generalization', arXiv:2504.02160). OmniContext is introduced in this report (Section 5.4; GPT-4.1 judge), so every cell is the authors' own run. 0-10 scale divided by 10.",
    "source_id": "report-2506.18871v4"
  },
  {
    "model": "omnigen-v1",
    "bench": "edival-o-turn3",
    "score": 0.2991,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'OmniGen' (release date column 24.09.11; Appendix M.5: 'OmniGen: repository at https://github.com/VectorSpaceLab/OmniGen', default settings); Table 3 EdiVal-O at turn 3 29.91 -> /100 (overall rank 12).",
    "source_id": "visual-edival"
  },
  {
    "model": "step1x-edit-v1-0",
    "bench": "edival-o-turn3",
    "score": 0.3898,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'Step1X-Edit' (release date column 25.04.25 = the v1.0 release; Appendix M.5: 'Step1X-Edit: repository at https://github.com/stepfun-ai/Step1X-Edit ; weights at https://huggingface.co/stepfun-ai/Step1X-Edit', default settings); Table 3 EdiVal-O at turn 3 38.98 -> /100 (overall rank 10). The visual-edival ledger scope already identifies this row as the v1.0 repo.",
    "source_id": "visual-edival"
  },
  {
    "model": "uno",
    "bench": "imagenworld-tig-human",
    "score": 0.78,
    "source": "https://arxiv.org/html/2603.27862v1#S5.T3",
    "by": "independent",
    "date": "2026-03-29",
    "note": "UNO; Table 3 Text-guided Image Generation (TIG) human overall 0.78±0.18 (mean±sd across images); default configuration, seed 42 for open models (Section 4). Table 2 lists 'UNO (Wu et al., 2025c)' as covering TIG, SRIG and MRIG; only the TIG row is used (no benchmark id for the reference-image tasks).",
    "source_id": "visual-imagenworld"
  },
  {
    "model": "step1x-edit-v1-0",
    "bench": "gedit-bench-en-overall",
    "score": 0.644,
    "source": "https://github.com/stepfun-ai/Step1X-Edit/blob/4cca05dfcd6c32dc810d0b4db84c2bf0682a5aeb/README.md",
    "by": "lab",
    "date": "2025-07-09",
    "note": "StepFun's own evaluation in the Step1X-Edit GitHub README (Apache-2.0) at commit 4cca05d, news item 'Jul 09, 2025', table introduced as 'Quantitative evaluation on GEdit-Bench-EN (Full set). G_SC, G_PQ, and G_O refer to the metrics evaluated by GPT-4.1, while Q_SC, Q_PQ, and Q_O refer to the metrics evaluated by Qwen2.5-VL-72B'; row 'Step1X-Edit (v1.0)', column G_O = 6.44 -> /10. Intermediate results released by the authors (huggingface.co/datasets/Shiyu95/gedit_results).",
    "source_id": "visual-step1x-edit-readme"
  },
  {
    "model": "step1x-edit-v1-1",
    "bench": "gedit-bench-en-overall",
    "score": 0.697,
    "source": "https://github.com/stepfun-ai/Step1X-Edit/blob/4cca05dfcd6c32dc810d0b4db84c2bf0682a5aeb/README.md",
    "by": "lab",
    "date": "2025-07-09",
    "note": "Same table as the v1.0 row (Step1X-Edit GitHub README @4cca05d, 'Jul 09, 2025' news item, GEdit-Bench-EN Full set, GPT-4.1 judge); row 'Step1X-Edit (v1.1)', column G_O = 6.97 -> /10. Same-maker sibling cell added so the v1.0 row is not alone in its run. NOTE: the existing gedit-bench-overall-lang-unspecified row for step1x-edit-v1-1 (0.697, v1p2 card) is the same measurement; see held_for_owner.",
    "source_id": "visual-step1x-edit-readme"
  },
  {
    "model": "cosmos3-super",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.893,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos3-Super', column 'Cosmos HUE / Text-to-Video' = 89.3 -> 89.3/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. NVIDIA's own model (mid-trained nvidia/Cosmos3-Super; Table 1 cell unstarred).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos3-super",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.896,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos3-Super', column 'Cosmos HUE / Image-to-Video' = 89.6 -> 89.6/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. NVIDIA's own model (mid-trained nvidia/Cosmos3-Super; Table 1 cell unstarred).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos3-nano",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.876,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos3-Nano', column 'Cosmos HUE / Text-to-Video' = 87.6 -> 87.6/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. NVIDIA's own model (nvidia/Cosmos3-Nano; Table 1 cell unstarred).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos3-nano",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.886,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos3-Nano', column 'Cosmos HUE / Image-to-Video' = 88.6 -> 88.6/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. NVIDIA's own model (nvidia/Cosmos3-Nano; Table 1 cell unstarred).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "wan-2-2-a14b",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.882,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Wan2.2-A14B', column 'Cosmos HUE / Text-to-Video' = 88.2 -> 88.2/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. T2V column mapped to the catalogued Wan2.2-T2V-A14B (the report names only 'Wan2.2-A14B').",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.884,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Wan2.2-A14B', column 'Cosmos HUE / Image-to-Video' = 88.4 -> 88.4/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. I2V column mapped to Wan2.2-I2V-A14B, the A14B checkpoint that takes an image (the report names only 'Wan2.2-A14B'; the T2V-A14B checkpoint has no image input).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanvideo-1-5",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.865,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'HunyuanVideo-1.5', column 'Cosmos HUE / Text-to-Video' = 86.5 -> 86.5/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. T2V column mapped to the catalogued HunyuanVideo 1.5 (T2V).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.856,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'HunyuanVideo-1.5', column 'Cosmos HUE / Image-to-Video' = 85.6 -> 85.6/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. I2V column mapped to the catalogued HunyuanVideo 1.5 (I2V).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "wan-2-1-14b",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.84,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Wan2.1-14B', column 'Cosmos HUE / Text-to-Video' = 84.0 -> 84.0/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. T2V column mapped to Wan2.1-T2V-14B; the I2V cell (Wan2.1-I2V-14B) is not mapped (checkpoint not catalogued).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.808,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Wan2.2-5B', column 'Cosmos HUE / Text-to-Video' = 80.8 -> 80.8/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Wan2.2-TI2V-5B handles both T2V and I2V.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.804,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Wan2.2-5B', column 'Cosmos HUE / Image-to-Video' = 80.4 -> 80.4/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Wan2.2-TI2V-5B handles both T2V and I2V.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "veo-3-1",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.913,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Veo-3.1', column 'Cosmos HUE / Text-to-Video' = 91.3 -> 91.3/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Closed-source; mapped to Veo 3.1 (standard, not Fast).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "veo-3-1",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.897,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Veo-3.1', column 'Cosmos HUE / Image-to-Video' = 89.7 -> 89.7/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Closed-source; mapped to Veo 3.1 (standard, not Fast).",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.9,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Seedance-1.5-Pro', column 'Cosmos HUE / Text-to-Video' = 90.0 -> 90.0/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Closed-source API model.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.876,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Seedance-1.5-Pro', column 'Cosmos HUE / Image-to-Video' = 87.6 -> 87.6/100. Cosmos HUE human evaluation on the PAIBench-G prompt set: atomic Yes/No/Unclear question verification by human annotators; score = share of Yes answers. Closed-source API model.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "sora-2",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.7994,
    "source": "https://arxiv.org/html/2512.09299v2#S3.T1",
    "by": "independent",
    "date": "2026-04-06",
    "note": "VABench v2 Table 1 (T2AV), row 'sora2', column 'Visual QA' = 0.7994. end-to-end AV model 'Sora2' via API. VABench paper (arXiv 2512.09299v2, CC BY 4.0) Sec. 4.1: 'Veo3-fast, Wan2.5 Preview, Sora2, Seedance-1.0-lite, Kling2.5 Turbo were accessed via APIs ... Wan2.2-TI2V ... deployed locally. Video outputs were set to 720P ... frame rate and duration following default settings.' Visual QA = mean over T2AV samples of the share of 3-7 prompt-derived visual detail questions the LLM judge marks as satisfied (Eq. 2).",
    "source_id": "visual-vabench"
  },
  {
    "model": "veo-3-fast",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.8095,
    "source": "https://arxiv.org/html/2512.09299v2#S3.T1",
    "by": "independent",
    "date": "2026-04-06",
    "note": "VABench v2 Table 1 (T2AV), row 'veo3', column 'Visual QA' = 0.8095. end-to-end AV model; the paper's abbreviation list reads 'veo3 (Veo3-Fast)', so mapped to Veo 3 Fast. VABench paper (arXiv 2512.09299v2, CC BY 4.0) Sec. 4.1: 'Veo3-fast, Wan2.5 Preview, Sora2, Seedance-1.0-lite, Kling2.5 Turbo were accessed via APIs ... Wan2.2-TI2V ... deployed locally. Video outputs were set to 720P ... frame rate and duration following default settings.' Visual QA = mean over T2AV samples of the share of 3-7 prompt-derived visual detail questions the LLM judge marks as satisfied (Eq. 2).",
    "source_id": "visual-vabench"
  },
  {
    "model": "wan-2-5-preview",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.7933,
    "source": "https://arxiv.org/html/2512.09299v2#S3.T1",
    "by": "independent",
    "date": "2026-04-06",
    "note": "VABench v2 Table 1 (T2AV), row 'wan2.5', column 'Visual QA' = 0.7933. end-to-end AV model; 'wan2.5 (Wan2.5 Preview)' via API. VABench paper (arXiv 2512.09299v2, CC BY 4.0) Sec. 4.1: 'Veo3-fast, Wan2.5 Preview, Sora2, Seedance-1.0-lite, Kling2.5 Turbo were accessed via APIs ... Wan2.2-TI2V ... deployed locally. Video outputs were set to 720P ... frame rate and duration following default settings.' Visual QA = mean over T2AV samples of the share of 3-7 prompt-derived visual detail questions the LLM judge marks as satisfied (Eq. 2).",
    "source_id": "visual-vabench"
  },
  {
    "model": "seedance-1-0-lite",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.7265,
    "source": "https://arxiv.org/html/2512.09299v2#S3.T1",
    "by": "independent",
    "date": "2026-04-06",
    "note": "VABench v2 Table 1 (T2AV), row 'seed_think / seed_mm', column 'Visual QA' = 0.7265. decoupled V+A pipeline whose video generator is Seedance-1.0-Lite (API, 720P); the Visual QA cell spans both audio pairings (rowspan=2 in the HTML table), so it measures the video generator alone. VABench paper (arXiv 2512.09299v2, CC BY 4.0) Sec. 4.1: 'Veo3-fast, Wan2.5 Preview, Sora2, Seedance-1.0-lite, Kling2.5 Turbo were accessed via APIs ... Wan2.2-TI2V ... deployed locally. Video outputs were set to 720P ... frame rate and duration following default settings.' Visual QA = mean over T2AV samples of the share of 3-7 prompt-derived visual detail questions the LLM judge marks as satisfied (Eq. 2).",
    "source_id": "visual-vabench"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.6363,
    "source": "https://arxiv.org/html/2512.09299v2#S3.T1",
    "by": "independent",
    "date": "2026-04-06",
    "note": "VABench v2 Table 1 (T2AV), row 'wan2.2_think / wan2.2_mm', column 'Visual QA' = 0.6363. decoupled V+A pipeline whose video generator is Wan2.2-TI2V (5B, run locally); the Visual QA cell spans both audio pairings (rowspan=2), so it measures the video generator alone. VABench paper (arXiv 2512.09299v2, CC BY 4.0) Sec. 4.1: 'Veo3-fast, Wan2.5 Preview, Sora2, Seedance-1.0-lite, Kling2.5 Turbo were accessed via APIs ... Wan2.2-TI2V ... deployed locally. Video outputs were set to 720P ... frame rate and duration following default settings.' Visual QA = mean over T2AV samples of the share of 3-7 prompt-derived visual detail questions the LLM judge marks as satisfied (Eq. 2).",
    "source_id": "visual-vabench"
  },
  {
    "model": "kandinsky-6-0-pro",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.652,
    "source": "https://arxiv.org/html/2610.05608v1#S7.T11",
    "by": "lab",
    "date": "2026-10-04",
    "note": "Kandinsky 6.0 report Table 11 'VABench results', column 'Kandinsky 6.0 Video Pro', row 'visual_qa' = 0.652. The authors' own model. Kandinsky 6.0 report (arXiv 2610.05608v1, CC BY 4.0) Sec. 7.1: 'We evaluate the final Kandinsky 6.0 Video checkpoints on VABench ... Results are reported for Kandinsky 6.0 Video Pro and Kandinsky 6.0 Video Lite after RL post-training, both at 512 × 768 resolution, alongside LTX 2.5 in its two-stage Full HD configuration.' Table 11 columns are VABench's T2AV metric set (speech_qn, lipsync present), so this is the T2AV track.",
    "source_id": "visual-kandinsky6-report"
  },
  {
    "model": "kandinsky-6-0-lite",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.691,
    "source": "https://arxiv.org/html/2610.05608v1#S7.T11",
    "by": "lab",
    "date": "2026-10-04",
    "note": "Kandinsky 6.0 report Table 11 'VABench results', column 'Kandinsky 6.0 Video Lite', row 'visual_qa' = 0.691. The authors' own model. Kandinsky 6.0 report (arXiv 2610.05608v1, CC BY 4.0) Sec. 7.1: 'We evaluate the final Kandinsky 6.0 Video checkpoints on VABench ... Results are reported for Kandinsky 6.0 Video Pro and Kandinsky 6.0 Video Lite after RL post-training, both at 512 × 768 resolution, alongside LTX 2.5 in its two-stage Full HD configuration.' Table 11 columns are VABench's T2AV metric set (speech_qn, lipsync present), so this is the T2AV track.",
    "source_id": "visual-kandinsky6-report"
  },
  {
    "model": "ltx-2-5",
    "bench": "vabench-t2av-visual-qa",
    "score": 0.617,
    "source": "https://arxiv.org/html/2610.05608v1#S7.T11",
    "by": "independent",
    "date": "2026-10-04",
    "note": "Kandinsky 6.0 report Table 11 'VABench results', column 'LTX 2.5', row 'visual_qa' = 0.617. Lightricks' open LTX-2.5 run by the Kandinsky authors in its two-stage Full HD configuration. Kandinsky 6.0 report (arXiv 2610.05608v1, CC BY 4.0) Sec. 7.1: 'We evaluate the final Kandinsky 6.0 Video checkpoints on VABench ... Results are reported for Kandinsky 6.0 Video Pro and Kandinsky 6.0 Video Lite after RL post-training, both at 512 × 768 resolution, alongside LTX 2.5 in its two-stage Full HD configuration.' Table 11 columns are VABench's T2AV metric set (speech_qn, lipsync present), so this is the T2AV track.",
    "source_id": "visual-kandinsky6-report"
  },
  {
    "model": "wan-3-0-prime",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1723.6,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Wan 3.0 Prime'; Table 2 human Bradley-Terry Elo 1723.6 (rank 1; 95% CI [1629.2, 1864.9], score rate 77.9%, 52 games); FAL API, reference-to-video (Table 1). Same table and run as the four WROP rows already imported; withheld in pass 'thin-visual' only because the tier was not catalogued.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "runway-aleph-2",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1518.3,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Runway Aleph 2'; Table 2 human Bradley-Terry Elo 1518.3 (rank 5; 95% CI [1425.1, 1621.6], score rate 52.9%, 51 games); Runway API, frame-wise edit/transfer (Table 1). The automatic-metric table marks it † n=297 (the Runway API refused some items). Same table and run as the four WROP rows already imported.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "claude-sonnet-5-5",
    "bench": "frontiermath-erdos",
    "score": 0.029,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'claude-sonnet-5-5_max'; run id bloom68-0ov6a268920re3sz; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.021. Row added to the file after the 2026-10-08 zip (run started 2026-10-08).",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "frontiermath-erdos",
    "score": 0.0294,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-09-24",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'claude-opus-5-5_max'; run id bloom68-opus55-m1pc5wbymin28jdj; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0206. Row added to the file after the 2026-10-08 zip.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gpt-5-5",
    "bench": "frontiermath-erdos",
    "score": 0,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_erdos.csv",
    "by": "independent",
    "date": "2026-08-28",
    "note": "Epoch AI internal evaluation; file frontiermath_erdos.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'gpt-5.5_xhigh'; run id bloom68-gpt55-4nh07xafb8lf7jqp; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0. Row already in the 2026-10-07/08 zips but not imported then (the earlier pass imported only its target models); unchanged in this snapshot.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "gpqa-diamond",
    "score": 0.8964646464646465,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-06-16",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id LuZUnhD5rBLjRw36DaqfvZ; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017473014662966007. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "swe-bench-verified",
    "score": 0.7763975155279503,
    "source": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv",
    "by": "independent",
    "date": "2026-06-18",
    "note": "Epoch AI internal evaluation; file swe_bench_verified.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id WUk3f44JhxtAJaSxi6gFZR; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.01897828052158487. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "otis-mock-aime",
    "score": 0.9666666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id HjTDENSpy9sancDzr7pccY; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02002208376727653. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "frontiermath",
    "score": 0.45263157894736844,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id KUar3FaFjzq5SMKrEhtn8e; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.029536098269922092. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "epoch-chess",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv",
    "by": "independent",
    "date": "2026-06-16",
    "note": "Epoch AI internal evaluation; file chess_puzzles.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id 8UqmhNg7pKbebcsYAZYpsh; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04020151261036849. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.024390243902439025,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-06-17",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-09 (zip sha256 a2381ce34ce1811b85da2c894cf09a84350a1439e35a583427a407724566bc35, ETag \"1c592e43476c33b3c09249035f1ecb05\"); exact model version 'deepseek-v4-pro_max'; run id RAZ6VMfGtQSE8EvUZRRz4Q; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.024390243902439025. Epoch model_metadata.csv: 'deepseek-v4-pro_max', model_group 'DeepSeek-V4-Pro', date 2026-04-24, display 'DeepSeek v4 Pro (max)', kept separate from 'deepseek-v4-pro-0813_*' (model_group 'DeepSeek V4 Pro 0813', date 2026-08-13, catalogued deepseek-v4-pro). Run started before the 0813 GA replaced the April preview under the API id deepseek-v4-pro on 2026-08-13, so it is the April 2026 preview checkpoint (catalogue deepseek-v4-pro-preview). Not imported earlier because the Epoch name 'deepseek-v4-pro' was unmapped.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.23913,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Claude-Opus-4.8 (max)', model_config 'anthropic/opus_48_max_128' (config file not present in eth-sri/matharena at b89f2f0ad64c; identity from published model_name only, same as the already-imported opus_5_max_128 and fable_5_max_128 rows of this month, eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.108696,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Gemini 3.1 Pro Preview', model_config 'gemini/gemini-31-pro' (plain API with MathArena Lean tools (configs/models/gemini/gemini-31-pro.yaml: gemini-3.1-pro-preview, thinking_level high, no harness), eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "glm-5-2",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.086957,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'GLM 5.2', model_config 'glm/glm-52' (plain API with MathArena Lean tools (configs/models/glm/glm-52.yaml: glm-5.2 via BigModel API, thinking enabled, reasoning_effort max, no harness), eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "step-3-7-flash",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'Step 3.7 Flash', model_config 'stepfun/3.7-flash' (plain API with MathArena Lean tools (configs/models/stepfun/3.7-flash.yaml: step-3.7-flash, reasoning_effort high, no harness); 0/46 proofs verified, recorded like the existing step-3-5-flash 0/41 March row, eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "matharena-arxivlean-2026-06",
    "score": 0.391304,
    "source": "https://huggingface.co/datasets/MathArena/arxivlean-0626_outputs/blob/4221f7241f21cf4d39a80b934b7ea76d53ba29ed/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivLean June 2026 outputs dataset (HF revision 4221f7241f21cf4d39a80b934b7ea76d53ba29ed); published model_name 'GPT-5.6-Sol (max, via Codex)', model_config 'openai/codex-pro-arxivlean' (Codex CLI harness (configs/models/openai/codex-pro-arxivlean.yaml: type codex_cli, model gpt-5.6-sol, model_reasoning_effort max, analysis_id openai/gpt-56-sol, ChatGPT/Codex subscription auth, web_search disabled, no network for launched commands), the same codex-harness treatment as the already-imported gpt-6-astra row of this month, eth-sri/matharena b89f2f0ad64c). 46/46 problems, 1-1 attempts per problem; score = mean Lean-verified proof success from the 'correct' column, averaged per problem then over problems. Lean v4.31.0 + mathlib; competition allows harness opt-in per model config (default_harness: false); max 200 tool calls (configs/competitions/arxivlean/june.yaml). Separate benchmark from earlier months. CC BY-SA 4.0.",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "qed-nano",
    "bench": "matharena-arxivmath-2025-12",
    "score": 0.264706,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-1225_outputs/blob/6e427c4860254adc93ee95ebd37dc2b6c6028b4a/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath December 2025 outputs dataset (HF revision 6e427c4860254adc93ee95ebd37dc2b6c6028b4a); published model_name 'QED-Nano', model_config 'other/qed-nano' (lm-provers/QED-Nano self-hosted (custom OpenAI-compatible endpoint), temperature 0.6, top_p 0.95, max_tokens 120000, per configs/models/other/qed-nano.yaml, eth-sri/matharena b89f2f0ad64c). 17/17 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "qed-nano",
    "bench": "matharena-arxivmath-2026-01",
    "score": 0.369565,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0126_outputs/blob/306b8e8d1e91c40beb6b2b003adb89e7951bd204/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath January 2026 outputs dataset (HF revision 306b8e8d1e91c40beb6b2b003adb89e7951bd204); published model_name 'QED-Nano', model_config 'other/qed-nano' (lm-provers/QED-Nano self-hosted (custom OpenAI-compatible endpoint), temperature 0.6, top_p 0.95, max_tokens 120000, per configs/models/other/qed-nano.yaml, eth-sri/matharena b89f2f0ad64c). 23/23 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "qed-nano",
    "bench": "matharena-arxivmath-2026-02",
    "score": 0.140625,
    "source": "https://huggingface.co/datasets/MathArena/arxivmath-0226_outputs/blob/92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8/data/train-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-09-11",
    "note": "MathArena ArXivMath February 2026 outputs dataset (HF revision 92e08f2fb94f5fd7fdbf02b335f5ce6ff9a589a8); published model_name 'QED-Nano', model_config 'other/qed-nano' (lm-provers/QED-Nano self-hosted (custom OpenAI-compatible endpoint), temperature 0.6, top_p 0.95, max_tokens 120000, per configs/models/other/qed-nano.yaml, eth-sri/matharena b89f2f0ad64c). 32/32 problems, 4 attempts per problem; score = mean final-answer correctness (parsed \\boxed{} answer vs gold), averaged per problem then over problems, computed by whatmodel.sh from the per-attempt 'correct' column. Competition config has no harness opt-in, so the plain API path was used (no tools). CC BY-SA 4.0 (share-alike tranche).",
    "source_id": "matharena-arxiv-outputs-bysa"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.11375,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-4o",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-4o' (API model in attempt metadata: 'gpt-4o' (OpenAI alias), run 2026-03-17, default sampling (no kwargs); mapped to gpt-4o-2024-08-06, the default snapshot of 'gpt-4o' on OpenAI's model page (as the catalogue entry and the inspect-charxiv 'openai/gpt-4o' rows)). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 400 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 45.5 / 400); method reproduces the published results.json exactly for every folder that has one. One attempt is null (failed call) and counts as unsolved; all 400 tasks and 419 test pairs present. Non-reasoning model, single complete run in this dataset. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-4o",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-4o' (API model in attempt metadata: 'gpt-4o' (OpenAI alias), run 2026-03-17, default sampling (no kwargs); mapped to gpt-4o-2024-08-06, the default snapshot of 'gpt-4o' on OpenAI's model page (as the catalogue entry and the inspect-charxiv 'openai/gpt-4o' rows)). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the 120 task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 0 / 120); method reproduces the published results.json exactly for every folder that has one. 0 of 167 test pairs solved; all 120 tasks present. Non-reasoning model, single complete run in this dataset. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "deepseek-v3-2-exp",
    "bench": "bfcl-v4",
    "score": 0.5412,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'DeepSeek-V3.2-Exp (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 54.12%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://api-docs.deepseek.com/news/news250528 (DeepSeek, MIT). FC entry chosen per the source rule; the archive also lists 'DeepSeek-V3.2-Exp (Prompt + Thinking)' = 56.73%, not used (one entry per model).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "grok-4-1-fast-non-reasoning",
    "bench": "bfcl-v4",
    "score": 0.5829,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Grok-4-1-fast-non-reasoning (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 58.29%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://docs.x.ai/docs/models (xAI). Distinct API model from the already-recorded 'Grok-4-1-fast-reasoning (FC)'. Needs the proposed catalogue entry grok-4-1-fast-non-reasoning.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "bfcl-v4",
    "score": 0.219,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Bielik-11B-v2.3-Instruct (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 21.90%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/speakleash/Bielik-11B-v2.3-Instruct (SpeakLeash & ACK Cyfronet AGH, Apache 2.0); Prompt is the only mode listed. Needs the proposed catalogue entry bielik-11b-v2-3-instruct.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "minicpm3-4b",
    "bench": "bfcl-v4",
    "score": 0.2555,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'MiniCPM3-4B-FC (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 25.55%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/openbmb/MiniCPM3-4B (openbmb, Apache-2.0); FC entry chosen over 'MiniCPM3-4B (Prompt)' = 22.08% per the source rule. Needs the proposed catalogue entry minicpm3-4b.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "bfcl-v3",
    "score": 0.7171,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4o-2024-11-20 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 71.71%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "bfcl-v3",
    "score": 0.7032,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4.5-Preview-2025-02-27 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 70.32%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-32b",
    "bench": "bfcl-v3",
    "score": 0.6925,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-32B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 69.25%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Same published entry name as the existing qwen3-32b bfcl-v4 row; it was missing from the earlier v3 import. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "bfcl-v3",
    "score": 0.6794,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-235B-A22B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 67.94%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Original April 2025 hybrid checkpoint (Model Link https://huggingface.co/Qwen/Qwen3-235B-A22B), not the 2507 Instruct/Thinking releases; thinking setting not recorded in the aggregate CSV. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.6572,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-30B-A3B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 65.72%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Original April 2025 hybrid checkpoint (Model Link https://huggingface.co/Qwen/Qwen3-30B-A3B), not the 2507 releases; thinking setting not recorded. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen3-4b",
    "bench": "bfcl-v3",
    "score": 0.6339,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Qwen3-4B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 63.39%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Original April 2025 hybrid checkpoint (Model Link https://huggingface.co/Qwen/Qwen3-4B), not the 2507 releases; thinking setting not recorded. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwq-32b",
    "bench": "bfcl-v3",
    "score": 0.583,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'QwQ-32B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 58.30%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Qwen/QwQ-32B; first-party-hosted run (the Novita-served entries in older snapshots not used). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "bfcl-v3",
    "score": 0.6471,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'DeepSeek-V3-0324 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 64.71%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://api-docs.deepseek.com/news/news250325. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "bfcl-v3",
    "score": 0.6379,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'DeepSeek-R1-0528 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 63.79%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://api-docs.deepseek.com/news/news250528. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "bfcl-v3",
    "score": 0.5608,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'o1-2024-12-17 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.08%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. FC entry per the source rule; the archive also lists 'o1-2024-12-17 (Prompt)' = 67.79%, not used. Reasoning effort not recorded in the aggregate CSV (API default). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "o3-mini",
    "bench": "bfcl-v3",
    "score": 0.5126,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'o3-mini-2025-01-31 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 51.26%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. FC entry per the source rule; 'o3-mini-2025-01-31 (Prompt)' = 64.61% not used. Reasoning effort not recorded in the aggregate CSV (API default). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "bfcl-v3",
    "score": 0.6398,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GPT-4o-mini-2024-07-18 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 63.98%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "grok-3-beta",
    "bench": "bfcl-v3",
    "score": 0.6331,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Grok-3-beta (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 63.31%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. The 2025-04-25 snapshot printed 55.31% for the same entry (re-run); the 2025-06-14 value is used. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "bfcl-v3",
    "score": 0.5305,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Grok-3-mini-beta (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 53.05%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Reasoning effort not recorded in the aggregate CSV. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "bfcl-v3",
    "score": 0.6122,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemini-2.0-Flash-001 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 61.22%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "bfcl-v3",
    "score": 0.5994,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemini-2.5-Pro-Preview-05-06 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 59.94%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-0-flash-thinking-exp-01-21",
    "bench": "bfcl-v3",
    "score": 0.5874,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemini-2.0-Flash-Thinking-Exp-01-21 (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 58.74%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt is the only mode listed for this model. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "bfcl-v3",
    "score": 0.5622,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Gemini-2.0-Flash-Lite-001 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.22%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. FC entry per the source rule ('Gemini-2.0-Flash-Lite-001 (Prompt)' = 56.39% not used). Needs the proposed comparison-only catalogue entry gemini-2-0-flash-lite-001. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "bfcl-v3",
    "score": 0.5858,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Claude-3.7-Sonnet-20250219 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 58.58%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Dated API snapshot; extended-thinking budget not recorded in the aggregate CSV. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "bfcl-v3",
    "score": 0.5695,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Claude-3-Opus-20240229 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.95%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "bfcl-v3",
    "score": 0.5688,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Claude-3.5-Sonnet-20241022 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.88%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "bfcl-v3",
    "score": 0.525,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'claude-3.5-haiku-20241022 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.50%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "bfcl-v3",
    "score": 0.5774,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Mistral-small-2503 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 57.74%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Mistral API model mistral-small-2503 = Mistral Small 3.1 24B (2503). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-lite",
    "bench": "bfcl-v3",
    "score": 0.5593,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Amazon-Nova-Lite-v1:0 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 55.93%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Same naming as the existing nova-pro / nova-micro rows (Amazon-Nova-*-v1:0). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "bfcl-v3",
    "score": 0.391,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'GLM-4-9b-Chat (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 39.10%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/THUDM/glm-4-9b-chat. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "phi-4-mini",
    "bench": "bfcl-v3",
    "score": 0.2816,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Phi-4-mini-instruct (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 28.16%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/microsoft/Phi-4-mini-instruct; FC entry per the source rule ('Phi-4-mini-instruct (Prompt)' = 31.27% not used). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "bfcl-v3",
    "score": 0.5357,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Llama-3.1-70B-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 53.57%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt entry chosen over 'Llama-3.1-70B-Instruct (FC)' = 27.12% to match the reviewed Llama-3.1 mode already used for llama-3-1-8b in this run ('Llama-3.1-8B-Instruct (Prompt)'); see held_for_owner. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "bfcl-v3",
    "score": 0.4798,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Command-R-Plus (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 47.98%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Result folder command-r-plus-FC, i.e. Cohere API alias 'command-r-plus', which Cohere's docs (cohere-developer-experience @a892550, command-r-plus.mdx) define as 'an alias for command-r-plus-04-2024'; Model Link https://txt.cohere.com/command-r-plus-microsoft-azure (April 2024 launch). Unchanged components since the 2024-12-29 snapshot. The earlier import skipped it only because the catalogue then lacked the 04-2024 checkpoint. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "bfcl-v3",
    "score": 0.4081,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Bielik-11B-v2.3-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 40.81%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/speakleash/Bielik-11B-v2.3-Instruct; Prompt is the only mode listed. Needs the proposed catalogue entry bielik-11b-v2-3-instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "minicpm3-4b",
    "bench": "bfcl-v3",
    "score": 0.503,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'MiniCPM3-4B-FC (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 50.30%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/openbmb/MiniCPM3-4B; FC entry chosen over 'MiniCPM3-4B (Prompt)' = 40.92% per the source rule. Needs the proposed catalogue entry minicpm3-4b. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "bfcl-v3",
    "score": 0.6337,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-72B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 63.37%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "bfcl-v3",
    "score": 0.6279,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-32B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 62.79%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-14b",
    "bench": "bfcl-v3",
    "score": 0.5871,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-14B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 58.71%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "bfcl-v3",
    "score": 0.5581,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-7B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 55.81%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "bfcl-v3",
    "score": 0.5037,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-3B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 50.37%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-1-5b",
    "bench": "bfcl-v3",
    "score": 0.4784,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-1.5B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 47.84%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "bfcl-v3",
    "score": 0.3563,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Qwen2.5-0.5B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 35.63%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-pro-exp-03-25",
    "bench": "bfcl-v3",
    "score": 0.6289,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Gemini-2.5-Pro-Exp-03-25 (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 62.89%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-v3",
    "bench": "bfcl-v3",
    "score": 0.5761,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'DeepSeek-V3 (FC)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 57.61%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://api-docs.deepseek.com/news/news1226 (December 2024 V3). Entry first appears in the 2024-12-29 snapshot with the same Non-Live Simple AST (78.67%) through 2025-04-25, i.e. outputs predate the V3-0324 release; the archive lists V3-0324 as a separate entry. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-r1",
    "bench": "bfcl-v3",
    "score": 0.5689,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'DeepSeek-R1 (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 56.89%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. January 2025 R1 (entry first listed 2025-03-17, before R1-0528; the archive lists DeepSeek-R1-0528 separately). Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "bfcl-v3",
    "score": 0.2066,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'QwQ-32B-Preview (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 20.66%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 93 of 94 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "bfcl-v3",
    "score": 0.6787,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'GPT-4-turbo-2024-04-09 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 67.87%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-large-2407",
    "bench": "bfcl-v3",
    "score": 0.5767,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'mistral-large-2407 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 57.67%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "bfcl-v3",
    "score": 0.5246,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'GPT-3.5-Turbo-0125 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 52.46%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-medium-2312",
    "bench": "bfcl-v3",
    "score": 0.5141,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Mistral-Medium-2312 (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 51.41%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-70b",
    "bench": "bfcl-v3",
    "score": 0.5083,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 50.83%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-8b",
    "bench": "bfcl-v3",
    "score": 0.3307,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 33.07%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores.  Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "bfcl-v3",
    "score": 0.4177,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Open-Mixtral-8x22b (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 41.77%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Mistral API open-mixtral-8x22b = Mixtral 8x22B Instruct v0.1; FC entry per the source rule ('Open-Mixtral-8x22b (Prompt)' = 49.51% not used). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "bfcl-v3",
    "score": 0.4221,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Open-Mixtral-8x7b (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 42.21%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Mistral API open-mixtral-8x7b = Mixtral 8x7B Instruct v0.1; Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-7b",
    "bench": "bfcl-v3",
    "score": 0.4131,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Qwen2-7B-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 41.31%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "bfcl-v3",
    "score": 0.2819,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Qwen2-1.5B-Instruct (Prompt)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 28.19%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Prompt is the only mode listed. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "bfcl-v3",
    "score": 0.2098,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'DeepSeek-Coder-V2-Lite-Instruct (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 20.98%. Entry absent from the 2025-06-14 snapshot used for this run, so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; of the entries common to both files, 74 of 76 have identical Overall Acc in 2025-06-14). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "codellama-7b",
    "bench": "humaneval",
    "score": 0.378,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-7B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval = 37.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 7B base checkpoint (catalogue codellama-7b, codellama/CodeLlama-7b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-7b",
    "bench": "humaneval-plus",
    "score": 0.354,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-7B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval+ = 35.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 7B base checkpoint (catalogue codellama-7b, codellama/CodeLlama-7b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-7b",
    "bench": "mbpp-plus",
    "score": 0.468,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-7B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 mbpp+ = 46.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 7B base checkpoint (catalogue codellama-7b, codellama/CodeLlama-7b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-13b",
    "bench": "humaneval",
    "score": 0.427,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-13B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval = 42.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 13B base checkpoint (codellama/CodeLlama-13b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-13b",
    "bench": "humaneval-plus",
    "score": 0.384,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-13B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval+ = 38.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 13B base checkpoint (codellama/CodeLlama-13b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-13b",
    "bench": "mbpp-plus",
    "score": 0.526,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-13B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 mbpp+ = 52.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 13B base checkpoint (codellama/CodeLlama-13b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-34b",
    "bench": "humaneval",
    "score": 0.518,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-34B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval = 51.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 34B base checkpoint (codellama/CodeLlama-34b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-34b",
    "bench": "humaneval-plus",
    "score": 0.439,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-34B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 humaneval+ = 43.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 34B base checkpoint (codellama/CodeLlama-34b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codellama-34b",
    "bench": "mbpp-plus",
    "score": 0.563,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeLlama-34B' (link https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/; base-model completion prompt); pass@1 mbpp+ = 56.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; Code Llama 34B base checkpoint (codellama/CodeLlama-34b-hf); link is Meta's Code Llama announcement. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-j-6b",
    "bench": "humaneval",
    "score": 0.122,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-J-6B' (link https://www.eleuther.ai/artifacts/gpt-j; base-model completion prompt); pass@1 humaneval = 12.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; link https://www.eleuther.ai/artifacts/gpt-j (EleutherAI/gpt-j-6b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-j-6b",
    "bench": "humaneval-plus",
    "score": 0.11,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-J-6B' (link https://www.eleuther.ai/artifacts/gpt-j; base-model completion prompt); pass@1 humaneval+ = 11.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; link https://www.eleuther.ai/artifacts/gpt-j (EleutherAI/gpt-j-6b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "humaneval",
    "score": 0.079,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-Neo-2.7B' (link https://www.eleuther.ai/artifacts/gpt-neo; base-model completion prompt); pass@1 humaneval = 7.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; link https://www.eleuther.ai/artifacts/gpt-neo (EleutherAI/gpt-neo-2.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "humaneval-plus",
    "score": 0.067,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-Neo-2.7B' (link https://www.eleuther.ai/artifacts/gpt-neo; base-model completion prompt); pass@1 humaneval+ = 6.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry; link https://www.eleuther.ai/artifacts/gpt-neo (EleutherAI/gpt-neo-2.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "humaneval",
    "score": 0.287,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B' (link https://mistral.ai/news/announcing-mistral-7b/; base-model completion prompt); pass@1 humaneval = 28.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry linking the Mistral 7B announcement (Sept 2023), i.e. Mistral-7B-v0.1 base, not an Instruct version. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "humaneval-plus",
    "score": 0.238,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B' (link https://mistral.ai/news/announcing-mistral-7b/; base-model completion prompt); pass@1 humaneval+ = 23.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry linking the Mistral 7B announcement (Sept 2023), i.e. Mistral-7B-v0.1 base, not an Instruct version. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "mbpp-plus",
    "score": 0.421,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-7B' (link https://mistral.ai/news/announcing-mistral-7b/; base-model completion prompt); pass@1 mbpp+ = 42.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Base (non-prompted) entry linking the Mistral 7B announcement (Sept 2023), i.e. Mistral-7B-v0.1 base, not an Instruct version. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "humaneval",
    "score": 0.89,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Pro 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 humaneval = 89.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-pro-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "humaneval-plus",
    "score": 0.793,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Pro 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 humaneval+ = 79.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-pro-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "mbpp-plus",
    "score": 0.746,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Pro 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 mbpp+ = 74.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-pro-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "humaneval",
    "score": 0.823,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Flash 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 humaneval = 82.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-flash-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "humaneval-plus",
    "score": 0.756,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Flash 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 humaneval+ = 75.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-flash-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "mbpp-plus",
    "score": 0.675,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Gemini 1.5 Flash 002' (link https://ai.google.dev/; chat/instruction prompt); pass@1 mbpp+ = 67.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Explicitly versioned API model gemini-1.5-flash-002. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "humaneval",
    "score": 0.872,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Claude Sonnet 3.5 (June 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval = 87.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. June 2024 Claude 3.5 Sonnet = the only June snapshot claude-3-5-sonnet-20240620 (the October model is a separate snapshot). The entry's link field (platform.openai.com) is a site error, not identity evidence. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "humaneval-plus",
    "score": 0.817,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Claude Sonnet 3.5 (June 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval+ = 81.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. June 2024 Claude 3.5 Sonnet = the only June snapshot claude-3-5-sonnet-20240620 (the October model is a separate snapshot). The entry's link field (platform.openai.com) is a site error, not identity evidence. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "mbpp-plus",
    "score": 0.743,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Claude Sonnet 3.5 (June 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 mbpp+ = 74.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. June 2024 Claude 3.5 Sonnet = the only June snapshot claude-3-5-sonnet-20240620 (the October model is a separate snapshot). The entry's link field (platform.openai.com) is a site error, not identity evidence. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-preview",
    "bench": "humaneval",
    "score": 0.963,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Preview (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval = 96.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-preview had a single snapshot (o1-preview-2024-09-12; the EvalPerf run file for the same entry is named o1-preview-2024-09-12). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-preview",
    "bench": "humaneval-plus",
    "score": 0.89,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Preview (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval+ = 89.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-preview had a single snapshot (o1-preview-2024-09-12; the EvalPerf run file for the same entry is named o1-preview-2024-09-12). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-preview",
    "bench": "mbpp-plus",
    "score": 0.802,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Preview (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 mbpp+ = 80.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-preview had a single snapshot (o1-preview-2024-09-12; the EvalPerf run file for the same entry is named o1-preview-2024-09-12). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-mini",
    "bench": "humaneval",
    "score": 0.963,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Mini (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval = 96.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-mini had a single snapshot (o1-mini-2024-09-12; matches the EvalPerf run file name). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-mini",
    "bench": "humaneval-plus",
    "score": 0.89,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Mini (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval+ = 89.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-mini had a single snapshot (o1-mini-2024-09-12; matches the EvalPerf run file name). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "o1-mini",
    "bench": "mbpp-plus",
    "score": 0.788,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'O1 Mini (Sept 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 mbpp+ = 78.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. o1-mini had a single snapshot (o1-mini-2024-09-12; matches the EvalPerf run file name). The o1 API does not accept temperature 0, so 'greedy' here means the API default sampling. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "humaneval",
    "score": 0.884,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT 4o Mini (July 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval = 88.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. gpt-4o-mini had a single snapshot (2024-07-18) at the time. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "humaneval-plus",
    "score": 0.835,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT 4o Mini (July 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 humaneval+ = 83.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. gpt-4o-mini had a single snapshot (2024-07-18) at the time. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "mbpp-plus",
    "score": 0.722,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT 4o Mini (July 2024)' (link https://platform.openai.com/docs/models/; chat/instruction prompt); pass@1 mbpp+ = 72.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. gpt-4o-mini had a single snapshot (2024-07-18) at the time. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "humaneval",
    "score": 0.902,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4-Turbo (April 2024)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval = 90.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. April 2024 GPT-4 Turbo GA = gpt-4-turbo-2024-04-09 (the gpt-4-turbo alias has pointed to it since release). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "humaneval-plus",
    "score": 0.866,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4-Turbo (April 2024)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval+ = 86.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. April 2024 GPT-4 Turbo GA = gpt-4-turbo-2024-04-09 (the gpt-4-turbo alias has pointed to it since release). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "humaneval",
    "score": 0.854,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval = 85.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-4 Turbo (linked DevDay post) = gpt-4-1106-preview. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "humaneval-plus",
    "score": 0.817,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval+ = 81.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-4 Turbo (linked DevDay post) = gpt-4-1106-preview. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "mbpp-plus",
    "score": 0.733,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 mbpp+ = 73.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-4 Turbo (linked DevDay post) = gpt-4-1106-preview. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-3.5-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval = 76.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-3.5 Turbo (linked DevDay post) = gpt-3.5-turbo-1106. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "humaneval-plus",
    "score": 0.707,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-3.5-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 humaneval+ = 70.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-3.5 Turbo (linked DevDay post) = gpt-3.5-turbo-1106. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "mbpp-plus",
    "score": 0.697,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-3.5-Turbo (Nov 2023)' (link https://openai.com/blog/new-models-and-developer-products-announced-at-devday; chat/instruction prompt); pass@1 mbpp+ = 69.7 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. November 2023 DevDay GPT-3.5 Turbo (linked DevDay post) = gpt-3.5-turbo-1106. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-0314",
    "bench": "humaneval",
    "score": 0.884,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4 (May 2023)' (link https://openai.com/research/gpt-4; chat/instruction prompt); pass@1 humaneval = 88.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Evaluated May 2023 (original EvalPlus paper run); the gpt-4 alias resolved to gpt-4-0314 until gpt-4-0613 shipped in June 2023 (temporal resolution of the alias). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-4-0314",
    "bench": "humaneval-plus",
    "score": 0.793,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-4 (May 2023)' (link https://openai.com/research/gpt-4; chat/instruction prompt); pass@1 humaneval+ = 79.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Evaluated May 2023 (original EvalPlus paper run); the gpt-4 alias resolved to gpt-4-0314 until gpt-4-0613 shipped in June 2023 (temporal resolution of the alias). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "humaneval",
    "score": 0.732,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-3.5 (May 2023)' (link https://openai.com/blog/chatgpt; chat/instruction prompt); pass@1 humaneval = 73.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Evaluated May 2023 (original EvalPlus paper run, link the ChatGPT post); the gpt-3.5-turbo alias resolved to gpt-3.5-turbo-0301 until the 0613 snapshot shipped in June 2023 (temporal resolution of the alias). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gpt-3-5-turbo-0301",
    "bench": "humaneval-plus",
    "score": 0.665,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'GPT-3.5 (May 2023)' (link https://openai.com/blog/chatgpt; chat/instruction prompt); pass@1 humaneval+ = 66.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Evaluated May 2023 (original EvalPlus paper run, link the ChatGPT post); the gpt-3.5-turbo alias resolved to gpt-3.5-turbo-0301 until the 0613 snapshot shipped in June 2023 (temporal resolution of the alias). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-large-2402",
    "bench": "humaneval",
    "score": 0.695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral Large (Mar 2024)' (link https://mistral.ai/news/mistral-large/; chat/instruction prompt); pass@1 humaneval = 69.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Mistral Large as of March 2024 (linked February 2024 launch post) = mistral-large-2402; Mistral Large 2 (2407) came later. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-large-2402",
    "bench": "humaneval-plus",
    "score": 0.622,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral Large (Mar 2024)' (link https://mistral.ai/news/mistral-large/; chat/instruction prompt); pass@1 humaneval+ = 62.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Mistral Large as of March 2024 (linked February 2024 launch post) = mistral-large-2402; Mistral Large 2 (2407) came later. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-large-2402",
    "bench": "mbpp-plus",
    "score": 0.595,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral Large (Mar 2024)' (link https://mistral.ai/news/mistral-large/; chat/instruction prompt); pass@1 mbpp+ = 59.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Mistral Large as of March 2024 (linked February 2024 launch post) = mistral-large-2402; Mistral Large 2 (2407) came later. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "humaneval",
    "score": 0.768,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-haiku (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval = 76.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Haiku had a single snapshot (claude-3-haiku-20240307). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "humaneval-plus",
    "score": 0.689,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-haiku (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval+ = 68.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Haiku had a single snapshot (claude-3-haiku-20240307). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "mbpp-plus",
    "score": 0.688,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-haiku (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 mbpp+ = 68.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Haiku had a single snapshot (claude-3-haiku-20240307). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "humaneval",
    "score": 0.829,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-opus (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval = 82.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Opus had a single snapshot (claude-3-opus-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "humaneval-plus",
    "score": 0.774,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-opus (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval+ = 77.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Opus had a single snapshot (claude-3-opus-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "mbpp-plus",
    "score": 0.733,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-opus (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 mbpp+ = 73.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Opus had a single snapshot (claude-3-opus-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "humaneval",
    "score": 0.707,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-sonnet (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval = 70.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Sonnet had a single snapshot (claude-3-sonnet-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "humaneval-plus",
    "score": 0.64,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-sonnet (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 humaneval+ = 64.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Sonnet had a single snapshot (claude-3-sonnet-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "mbpp-plus",
    "score": 0.693,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-3-sonnet (Mar 2024)' (link https://www.anthropic.com/news/claude-3-family; chat/instruction prompt); pass@1 mbpp+ = 69.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Claude 3 Sonnet had a single snapshot (claude-3-sonnet-20240229). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "evalperf-correctness-20240328",
    "score": 0.6983898305084746,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/ise-uiuc--Magicoder-S-DS-6.7B_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "date": "2024-10-19",
    "note": "Published entry 'ise-uiuc--Magicoder-S-DS-6.7B'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export. Run file date 2024-10-19 16:28. Catalogue magicoder-s-ds-6-7b (ise-uiuc/Magicoder-S-DS-6.7B).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "cruxeval-zeroeval",
    "score": 0.68,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-pro' resolved temporally: the raw result files were last changed (result_dirs/crux/gemini-1.5-pro.json in commit 2ea30ff (2024-08-02)), before gemini-1.5-pro-002 was released (2024-09-24), when gemini-1.5-pro-001 was the only stable 1.5 Pro version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "cruxeval-zeroeval",
    "score": 0.6375,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-flash' resolved temporally: the raw result files were last changed (result_dirs/crux/gemini-1.5-flash.json in commit 24b104d (2024-07-31)), before gemini-1.5-flash-002 was released (2024-09-24), when gemini-1.5-flash-001 was the only stable 1.5 Flash version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "gsm8k-zeroeval",
    "score": 0.934,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-pro' resolved temporally: the raw result files were last changed (result_dirs/gsm/gemini-1.5-pro.json in commit cbfbc4e (2024-07-14)), before gemini-1.5-pro-002 was released (2024-09-24), when gemini-1.5-pro-001 was the only stable 1.5 Pro version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "gsm8k-zeroeval",
    "score": 0.9136,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-flash' resolved temporally: the raw result files were last changed (result_dirs/gsm/gemini-1.5-flash.json in commit cbfbc4e (2024-07-14)), before gemini-1.5-flash-002 was released (2024-09-24), when gemini-1.5-flash-001 was the only stable 1.5 Flash version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "gsm8k-zeroeval",
    "score": 0.8014,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r-plus'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r-plus', which Cohere's docs (cohere-developer-experience @a892550, command-r-plus.mdx) define as 'an alias for command-r-plus-04-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-03-2024",
    "bench": "gsm8k-zeroeval",
    "score": 0.5299,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r', which Cohere's docs (cohere-developer-experience @a892550, command-r.mdx) define as 'an alias for command-r-03-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8276,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.94. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-pro' resolved temporally: the raw result files were last changed (result_dirs/mmlu-redux/gemini-1.5-pro.json in commit e26c69c (2024-07-13)), before gemini-1.5-pro-002 was released (2024-09-24), when gemini-1.5-pro-001 was the only stable 1.5 Pro version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7736,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.26. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-flash' resolved temporally: the raw result files were last changed (result_dirs/mmlu-redux/gemini-1.5-flash.json in commit e26c69c (2024-07-13)), before gemini-1.5-flash-002 was released (2024-09-24), when gemini-1.5-flash-001 was the only stable 1.5 Flash version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6861,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r-plus'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r-plus', which Cohere's docs (cohere-developer-experience @a892550, command-r-plus.mdx) define as 'an alias for command-r-plus-04-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-03-2024",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.6112,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.04. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r', which Cohere's docs (cohere-developer-experience @a892550, command-r.mdx) define as 'an alias for command-r-03-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "zebralogic-zeroeval",
    "score": 0.194,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=22.70. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-flash' resolved temporally: the raw result files were last changed (result_dirs/zebra-grid/gemini-1.5-flash.json in commit 428410d (2024-07-12)), before gemini-1.5-flash-002 was released (2024-09-24), when gemini-1.5-flash-001 was the only stable 1.5 Flash version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "zebralogic-zeroeval",
    "score": 0.194,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.80. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-pro' resolved temporally: the raw result files were last changed (result_dirs/zebra-grid/gemini-1.5-pro.json in commit 428410d (2024-07-12)), before gemini-1.5-pro-002 was released (2024-09-24), when gemini-1.5-pro-001 was the only stable 1.5 Pro version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "zebralogic-zeroeval",
    "score": 0.139,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r-plus'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r-plus', which Cohere's docs (cohere-developer-experience @a892550, command-r-plus.mdx) define as 'an alias for command-r-plus-04-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "command-r-03-2024",
    "bench": "zebralogic-zeroeval",
    "score": 0.099,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'command-r'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Cohere API alias 'command-r', which Cohere's docs (cohere-developer-experience @a892550, command-r.mdx) define as 'an alias for command-r-03-2024'; the raw result files were added in commit 43d7919 (2024-07-22), before the 08-2024 models existed.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-mini",
    "bench": "math-l5-zeroeval",
    "score": 0.8932,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-mini-2024-09-12'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=5.27. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "o1-preview",
    "bench": "math-l5-zeroeval",
    "score": 0.8447,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'o1-preview-2024-09-12'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=3.33. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "grok-2-1212",
    "bench": "math-l5-zeroeval",
    "score": 0.6089,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'grok-2-1212'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "math-l5-zeroeval",
    "score": 0.6019,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-72B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.53. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "math-l5-zeroeval",
    "score": 0.595,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-32B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "math-l5-zeroeval",
    "score": 0.5936,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20241022'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "math-l5-zeroeval",
    "score": 0.5534,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-08-06'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=4.02. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "math-l5-zeroeval",
    "score": 0.5479,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-2024-05-13'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=4.58. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "math-l5-zeroeval",
    "score": 0.5215,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4o-mini-2024-07-18'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "math-l5-zeroeval",
    "score": 0.5187,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-sonnet-20240620'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "math-l5-zeroeval",
    "score": 0.5146,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-7B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=2.77. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "math-l5-zeroeval",
    "score": 0.4854,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Mistral-Large-2'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.69. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "math-l5-zeroeval",
    "score": 0.4646,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-5-haiku-20241022'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "math-l5-zeroeval",
    "score": 0.4646,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-turbo-2024-04-09'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.55. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "math-l5-zeroeval",
    "score": 0.4466,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2.5-0908'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.66. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "math-l5-zeroeval",
    "score": 0.4313,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-70B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=12.21. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "math-l5-zeroeval",
    "score": 0.3981,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.42. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-pro' resolved temporally: the raw result files were last changed (result_dirs/math-l5/gemini-1.5-pro.json in commit 8b70f64 (2024-09-09)), before gemini-1.5-pro-002 was released (2024-09-24), when gemini-1.5-pro-001 was the only stable 1.5 Pro version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "math-l5-zeroeval",
    "score": 0.3828,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-72B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "math-l5-zeroeval",
    "score": 0.3689,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-opus-20240229'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "math-l5-zeroeval",
    "score": 0.3481,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=2.36. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. API alias 'gemini-1.5-flash' resolved temporally: the raw result files were last changed (result_dirs/math-l5/gemini-1.5-flash.json in commit 8514734 (2024-09-10)), before gemini-1.5-flash-002 was released (2024-09-24), when gemini-1.5-flash-001 was the only stable 1.5 Flash version.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-27b",
    "bench": "math-l5-zeroeval",
    "score": 0.2663,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-27b-it'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.42. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-4-0314",
    "bench": "math-l5-zeroeval",
    "score": 0.2607,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-4-0314'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-5-3b",
    "bench": "math-l5-zeroeval",
    "score": 0.2552,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2.5-3B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=10.26. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-70b",
    "bench": "math-l5-zeroeval",
    "score": 0.251,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-70B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.53. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "qwen2-7b",
    "bench": "math-l5-zeroeval",
    "score": 0.2386,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Qwen2-7B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=3.61. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "math-l5-zeroeval",
    "score": 0.2219,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3.1-8B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=19.42. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "athene-70b",
    "bench": "math-l5-zeroeval",
    "score": 0.2067,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Athene-70B'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.55. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "math-l5-zeroeval",
    "score": 0.1997,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Yi-1.5-9B-Chat'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=14.01. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-9b",
    "bench": "math-l5-zeroeval",
    "score": 0.1942,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-9b-it'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.69. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "math-l5-zeroeval",
    "score": 0.1872,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3.5-mini-instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=6.24. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "math-l5-zeroeval",
    "score": 0.1817,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Yi-1.5-34B-Chat'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.42. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "math-l5-zeroeval",
    "score": 0.1623,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Phi-3-mini-4k-instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=9.15. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "math-l5-zeroeval",
    "score": 0.1512,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'claude-3-haiku-20240307'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "math-l5-zeroeval",
    "score": 0.1373,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gpt-3.5-turbo-0125'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=0.42. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-8b",
    "bench": "math-l5-zeroeval",
    "score": 0.0791,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'Meta-Llama-3-8B-Instruct'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=11.93. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemma-2-2b",
    "bench": "math-l5-zeroeval",
    "score": 0.043,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemma-2-2b-it'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=16.23. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. For 405B use Hyperbolic consistently, not the highest provider score. Snapshot committed 2025-04-14; inference dates unspecified. Model mapping reuses the reviewed mapping of the same entry name in the existing ZeroEval rows.",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "openthinker2-32b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.032,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 'OpenThinker2-32B' = 3.2% (CI --0.3/++0.3); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01. Open checkpoint open-thoughts/OpenThinker2-32B (row label is the HF repo name; Qwen2.5-32B-Instruct fine-tune); reasoning model, thinking always on. Needs the proposed catalogue entry openthinker2-32b.",
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "openthinker2-32b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.236,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 'OpenThinker2-32B' = 23.6% (CI --1.5/++1.3); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01. Open checkpoint open-thoughts/OpenThinker2-32B (row label is the HF repo name; Qwen2.5-32B-Instruct fine-tune); reasoning model, thinking always on. Needs the proposed catalogue entry openthinker2-32b.",
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "s1-1-32b",
    "bench": "arena-hard-v2-hard-prompt-sc-gemini25",
    "score": 0.223,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Hard Prompt, Style Control, and Gemini-2.5 as Judge'; row 's1.1-32B' = 22.3% (CI --1.7/++1.5); win rate vs baseline o3-mini-2025-01-31 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01. Open checkpoint simplescaling/s1.1-32B (row label is the HF repo name); default decoding, no budget-forcing setting stated. Needs the proposed catalogue entry s1-1-32b.",
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "s1-1-32b",
    "bench": "arena-hard-v2-creative-writing",
    "score": 0.082,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Arena-Hard-Auto README, 'Arena-Hard-v2.0-Preview' leaderboard block 'Creative Writing, Ensemble GPT-4.1 and Gemini 2.5 as Judges'; row 's1.1-32B' = 8.2% (CI printed truncated as '(-0.9 / +0.' at the end of the README code block); win rate vs baseline gemini-2.0-flash-001 (50%). Run by the LMArena/Arena-Hard maintainers; block added 2025-05-01. Open checkpoint simplescaling/s1.1-32B (row label is the HF repo name); default decoding, no budget-forcing setting stated. Needs the proposed catalogue entry s1-1-32b.",
    "source_id": "arena-hard-auto-readme-v2"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 30.3636,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'reka-flash-20240226': WB_score.task_macro = 30.3636 (WB_score 34.6041; raw mean checklist score 6.7302 over 1023 tasks in score.json). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Served by the Reka API: scripts/reka-flash-20240226.sh at the same commit sets model_name=\"reka/reka-flash-20240226\" (engine reka, temperature 0, max_tokens 4096), the dated Reka Flash API snapshot that LMArena lists as reka-flash-21b-20240226. WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date.",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "scicode-main-official",
    "score": 0.031,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Deepseek-Coder-v2'; Main Problem Resolve Rate 3.1% (subproblem 21.2% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same Main/Subproblem values (Table 'Subproblem / Main Problem': Deepseek-Coder-v2 21.2 / 3.1) and describes it in §3.1 as 'Deepseek-Coder-v2 [102]: Mixture-of-Experts (MoE) code language model continue pre-trained on DeepSeek-V2', citing the DeepSeek-Coder-V2 paper; evaluated before 2024-07-18, when the June 2024 release (DeepSeek-Coder-V2-Instruct, 0614) was the only full-size DeepSeek-Coder-V2 instruct model (the 0724 update came on 2024-07-24; the Lite model is never named).",
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "aider-code-editing",
    "score": 0.316,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-11",
    "note": "Published entry 'Qwen2.5-Coder-1.5B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 31.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=bb5681c; command=aider --model openai/Qwen2.5-Coder-1.5B-Instruct; run=2024-11-11-19-37-01--Qwen2.5-Coder-1.5B-Instruct. OpenAI-compatible endpoint serving Qwen2.5-Coder-1.5B-Instruct (not the quantised Ollama/GGUF entries); run after the model's public release (HF repo created 2024-09-18). Apache-2.0 (repository LICENSE.txt). Needs the proposed catalogue entry qwen2-5-coder-1-5b.",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "openthinker2-32b",
    "bench": "aime-2024",
    "score": 0.767,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'OpenThinker2-32B', column 'AIME24' = 76.7. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Mean accuracy reported by Evalchemy (single-response pass@1 averaged over repeated runs; no majority vote stated). Checkpoint: this repo (open-thoughts/OpenThinker2-32B @0aefafe). Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker2-32b",
    "bench": "aime-2025",
    "score": 0.587,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'OpenThinker2-32B', column 'AIME25' = 58.7. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' AIME25 = the full 2025 AIME (I and II) as packaged by Evalchemy. Checkpoint: this repo (open-thoughts/OpenThinker2-32B @0aefafe). Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker2-32b",
    "bench": "math-500",
    "score": 0.908,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'OpenThinker2-32B', column 'MATH500' = 90.8. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: this repo (open-thoughts/OpenThinker2-32B @0aefafe). Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker2-32b",
    "bench": "gpqa-diamond",
    "score": 0.641,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'OpenThinker2-32B', column 'GPQA-D' = 64.1. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: this repo (open-thoughts/OpenThinker2-32B @0aefafe). Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "aime-2024",
    "score": 0.747,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'DeepSeek-R1-Distill-Qwen-32B', column 'AIME24' = 74.7. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Mean accuracy reported by Evalchemy (single-response pass@1 averaged over repeated runs; no majority vote stated). Checkpoint: row links https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "aime-2025",
    "score": 0.5,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'DeepSeek-R1-Distill-Qwen-32B', column 'AIME25' = 50.0. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' AIME25 = the full 2025 AIME (I and II) as packaged by Evalchemy. Checkpoint: row links https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "math-500",
    "score": 0.9,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'DeepSeek-R1-Distill-Qwen-32B', column 'MATH500' = 90.0. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: row links https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "gpqa-diamond",
    "score": 0.658,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'DeepSeek-R1-Distill-Qwen-32B', column 'GPQA-D' = 65.8. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: row links https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "aime-2024",
    "score": 0.593,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'S1.1-32B', column 'AIME24' = 59.3. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Mean accuracy reported by Evalchemy (single-response pass@1 averaged over repeated runs; no majority vote stated). Checkpoint: row links https://huggingface.co/simplescaling/s1.1-32B (no budget forcing stated here, unlike the s1.1 card). Needs the proposed catalogue entry s1-1-32b. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "aime-2025",
    "score": 0.427,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'S1.1-32B', column 'AIME25' = 42.7. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' AIME25 = the full 2025 AIME (I and II) as packaged by Evalchemy. Checkpoint: row links https://huggingface.co/simplescaling/s1.1-32B (no budget forcing stated here, unlike the s1.1 card). Needs the proposed catalogue entry s1-1-32b. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "math-500",
    "score": 0.874,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'S1.1-32B', column 'MATH500' = 87.4. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: row links https://huggingface.co/simplescaling/s1.1-32B (no budget forcing stated here, unlike the s1.1 card). Needs the proposed catalogue entry s1-1-32b. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "gpqa-diamond",
    "score": 0.62,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table, row 'S1.1-32B', column 'GPQA-D' = 62.0. Open Thoughts' own run: the card states 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' Checkpoint: row links https://huggingface.co/simplescaling/s1.1-32B (no budget forcing stated here, unlike the s1.1 card). Needs the proposed catalogue entry s1-1-32b. Other columns (AMC23, LCBv2) have no matching benchmark id and are not used; Light-R1-32B and OpenThinker-32B rows are not catalogued.",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vector-inspect-gpqa-diamond",
    "score": 0.510101,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o', task 'gpqa_diamond', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.eval). Unversioned alias 'gpt-4o'; the linked Inspect log (storage.googleapis.com/inspect-evals/base/eval/gpt-4o/logs/2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.eval) records eval model 'openai/gpt-4o' and every sampled API response's output.model 'gpt-4o-2024-08-06' (checked 2026-10-09), i.e. the 2024-08-06 snapshot. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vector-inspect-mmlu-pro",
    "score": 0.745096,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o', task 'mmlu_pro', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.eval). Unversioned alias 'gpt-4o'; the linked Inspect log (storage.googleapis.com/inspect-evals/base/eval/gpt-4o/logs/2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.eval) records eval model 'openai/gpt-4o' and every sampled API response's output.model 'gpt-4o-2024-08-06' (checked 2026-10-09), i.e. the 2024-08-06 snapshot. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vector-inspect-math",
    "score": 0.7054,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o', task 'math', metric 'accuracy'; run with Inspect Evals (log logs/2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.eval). Unversioned alias 'gpt-4o'; the linked Inspect log (storage.googleapis.com/inspect-evals/base/eval/gpt-4o/logs/2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.eval) records eval model 'openai/gpt-4o' and every sampled API response's output.model 'gpt-4o-2024-08-06' (checked 2026-10-09), i.e. the 2024-08-06 snapshot. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vector-inspect-humaneval",
    "score": 0.908537,
    "source": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json",
    "by": "independent",
    "date": "2024-10-31",
    "note": "Vector Institute Evaluation Leaderboard entry 'gpt-4o', task 'humaneval', metric 'mean'; run with Inspect Evals (log logs/2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.eval). Unversioned alias 'gpt-4o'; the linked Inspect log (storage.googleapis.com/inspect-evals/base/eval/gpt-4o/logs/2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.eval) records eval model 'openai/gpt-4o' and every sampled API response's output.model 'gpt-4o-2024-08-06' (checked 2026-10-09), i.e. the 2024-08-06 snapshot. Date is the Inspect log date. Space revision f348507a79f56dae8624b525574cb326359a20c8.",
    "source_id": "vector-inspect-evals-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.638,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o/miniwob.json",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 63.8% +/- 1.9 (divided by 100); study 2024-10-25_06-08-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'GPT-4o and GPT-4o Mini ... using the Azure OpenAI API ... We use respectively checkpoints 2024-08-06 and 2024-07-18'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.314,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o/webarena.json",
    "by": "independent",
    "date": "2024-11-29",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 31.4% +/- 1.6 (divided by 100); study d2eed215-91bb-4603-b69c-8ef8f9d57f34; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses GPT-4o as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'GPT-4o and GPT-4o Mini ... using the Azure OpenAI API ... We use respectively checkpoints 2024-08-06 and 2024-07-18'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.455,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o/workarena-l1.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 45.5% +/- 2.7 (divided by 100); study 2024-10-23_14-17-40; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'GPT-4o and GPT-4o Mini ... using the Azure OpenAI API ... We use respectively checkpoints 2024-08-06 and 2024-07-18'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.085,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-GPT-4o/workarena-l2.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-GPT-4o' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 8.5% +/- 1.8 (divided by 100); study 2024-10-23_17-10-46; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses GPT-4o as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'GPT-4o and GPT-4o Mini ... using the Azure OpenAI API ... We use respectively checkpoints 2024-08-06 and 2024-07-18'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "browsergym-genericagent-miniwob",
    "score": 0.698,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-3.5-Sonnet/miniwob.json",
    "by": "independent",
    "date": "2024-10-25",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-3.5-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on MiniWoB; success rate 69.8% +/- 1.8 (divided by 100); study 2024-10-25_06-08-16; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses Claude-3.5-sonnet as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'Claude 3.5 Sonnet ... using the OpenRouter API ... We use the 2024-10-22 checkpoint, most recent to this day'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "browsergym-genericagent-webarena",
    "score": 0.362,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-3.5-Sonnet/webarena.json",
    "by": "independent",
    "date": "2024-11-29",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-3.5-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WebArena; success rate 36.2% +/- 1.7 (divided by 100); study b5fc5be7-54cc-4fc1-a9ee-73447b9c3eae; benchmark_tuned=No, followed_evaluation_protocol=Yes. Folder README: 'It uses Claude-3.5-sonnet as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'Claude 3.5 Sonnet ... using the OpenRouter API ... We use the 2024-10-22 checkpoint, most recent to this day'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "browsergym-genericagent-workarena-l1",
    "score": 0.564,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-3.5-Sonnet/workarena-l1.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-3.5-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L1; success rate 56.4% +/- 2.7 (divided by 100); study 2024-10-23_14-17-40; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses Claude-3.5-sonnet as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'Claude 3.5 Sonnet ... using the OpenRouter API ... We use the 2024-10-22 checkpoint, most recent to this day'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "browsergym-genericagent-workarena-l2",
    "score": 0.391,
    "source": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/294ebe1dc36ba83c6916c64c64fb870af3dc8cee/results/GenericAgent-Claude-3.5-Sonnet/workarena-l2.json",
    "by": "independent",
    "date": "2024-10-23",
    "note": "BrowserGym leaderboard, agent 'GenericAgent-Claude-3.5-Sonnet' (ServiceNow AgentLab GenericAgent, TMLR config flags, AXTree observation, no screenshots) on WorkArena-L2; success rate 39.1% +/- 3.2 (divided by 100); study 2024-10-23_17-10-46; benchmark_tuned=No, followed_evaluation_protocol=Yes. date_time field is the placeholder '2021-01-01 12:00:00'; date taken from the study id. Folder README: 'It uses Claude-3.5-sonnet as a backend' (unversioned); checkpoint from the same authors' BrowserGym paper (arXiv 2412.05467v4, Sec. 4: 'Claude 3.5 Sonnet ... using the OpenRouter API ... We use the 2024-10-22 checkpoint, most recent to this day'), whose Table 2 prints the identical value and standard error for this agent (identity evidence only; numbers taken from the Space). Agent scaffold result, not a bare-model score.",
    "source_id": "servicenow-browsergym-leaderboard"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "webapp1k-react-pass1",
    "score": 0.6813,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'gemini-1.5-pro' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-07-22 (commit e7c3bd05 'Create results.csv') with an unchanged value in every later revision; gemini-1.5-pro-001 was the only Gemini 1.5 Pro GA version then (-002 released 2024-09-24, Gemini API changelog). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "webapp1k-react-pass1",
    "score": 0.57,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results.csv",
    "by": "independent",
    "date": "2024-07-22",
    "note": "Entry 'gemini-1.5-flash' in results.csv: mean per-problem Successes/Runs over 1000 problems (10000 runs), reproducing the Space's pass@1. First published 2024-07-22 (commit e7c3bd05 'Create results.csv') with an unchanged value in every later revision; gemini-1.5-flash-001 was the only Gemini 1.5 Flash GA version then (-002 released 2024-09-24; the gemini-1.5-flash alias moved to -002 on 2024-11-21, Gemini API changelog). Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-preview-03-25",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.587,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-02",
    "note": "Entry 'gemini-2.5-pro-preview-03-25' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-05-02 (commit 1af8751d) with an unchanged value in every later revision, i.e. before the Gemini API changelog's 2025-05-06 note that gemini-2.5-pro-preview-03-25 'will automatically point to' 05-06, so this is the 03-25 preview. Needs the proposed comparison-only model gemini-2-5-pro-preview-03-25. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gpt-5-codex",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.75,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-11-22",
    "note": "Entry 'gpt-5-codex' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. First published 2025-11-22 (commit 615476a6), unchanged at the pinned revision. OpenAI API model gpt-5-codex (launched 2025-09-23; undated alias, OpenAI does not list dated snapshots). Needs the proposed comparison-only model gpt-5-codex. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history; value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "gemini-2-5-pro-preview-03-25",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.941,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.5-pro-preview-03-25'; column 'Model Avg' = 0.941; single-turn 0.93, multi-turn 0.95; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Row first published in commit 23236a91 ('added gemini 2.5 and llama 4', 2025-04-07) with the same value, before the 2025-05-06 repointing of the preview-03-25 id, so this is the 03-25 preview. Needs the proposed comparison-only model gemini-2-5-pro-preview-03-25.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "terminal-bench-4",
    "score": 0.2364,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-opus-4-8' (display 'Opus 4.8'), agent Claude Code (claude-code 2.1.231), reasoning_effort max. metrics.accuracy 23.64% ± 3.56 (95% CI half-width); 78/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-05-28 is the model release date. file sha256 25aea019c65af71083bb684cd26c30c4c6eb3f0c91d4d7473c7090c1b5e1d289. Added in the pass 8 refresh: the commit is unchanged (no leaderboard/submissions change between 8d0eb80 and 209679e, 2026-10-09); the earlier import covered only its target models.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "terminal-bench-4",
    "score": 0.1242,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-08-26-anthropic-claude-sonnet-5-max-claude-code.json",
    "by": "independent",
    "date": "2026-08-26",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'anthropic/claude-sonnet-5' (display 'Sonnet 5'), agent Claude Code (claude-code 2.1.231), reasoning_effort max. metrics.accuracy 12.42% ± 3.06 (95% CI half-width); 41/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-06-30 is the model release date. file sha256 f36c3c2fc771001978ceb3c8acd6c86302808dd837411fb4c71ced8f2c0fd80d. Added in the pass 8 refresh: the commit is unchanged (no leaderboard/submissions change between 8d0eb80 and 209679e, 2026-10-09); the earlier import covered only its target models.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "terminal-bench-4",
    "score": 0.3576,
    "source": "https://github.com/harbor-framework/terminal-bench/blob/8d0eb80ae63971cdcc89b37a672a3bfe2a6184a1/leaderboard/submissions/2026-10-06-z-ai-glm-5-3-flash-none-claude-code.json",
    "by": "independent",
    "date": "2026-10-06",
    "note": "Official Terminal-Bench 4.0 leaderboard submission file (dataset terminal-bench/terminal-bench ref v4.0.0, maintainer-run; community submissions closed). model_name 'z-ai/glm-5.3-flash' (display 'GLM-5.3-Flash'), agent Claude Code (claude-code 2.1.285), reasoning_effort None. metrics.accuracy 35.76% ± 3.54 (95% CI half-width); 118/330 trials (5 trials per task), 0 disqualified. Harness differs by vendor (agent per note), as the existing terminal-bench-4 description allows. Date = submission file date; metadata.date 2026-08-26 is the model release date. file sha256 a51d1277c9a54a4276ac547c33288b8163e5f771462d37edb2eb69eec322ed25. Added in the pass 8 refresh: the commit is unchanged (no leaderboard/submissions change between 8d0eb80 and 209679e, 2026-10-09); the earlier import covered only its target models.",
    "source_id": "terminal-bench-leaderboard-submissions"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "tau2-retail",
    "score": 0.7939,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/submission.json",
    "by": "lab",
    "date": "2026-01-26",
    "note": "Qwen's own submission (submitting_organization 'Qwen', submission_date 2026-01-26), model_name 'Qwen3-Max-Thinking'; results.retail.pass_1 = 79.3859649122807 (rounded to 4 dp; = 362/456 = 114 tasks x 4 trials). methodology: tau2_bench_version v0.1.3, user_simulator gpt-4.1-2025-04-14, modified_prompts false, omitted_questions false, 'Evaluation conducted using standard tau2-bench protocol'. Checkpoint: the trajectory file committed with the submission at 70e700c (web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/trajectories/retail_llm_agent_qwen3-max-2026-01-23_user_simulator_gpt-4.1-2025-04-14.json, removed from git in 20062c7) records agent llm 'qwen3-max-2026-01-23', temperature 0.0, 4 trials, max_steps 200; re-computed pass rate 0.79386. Thinking flag not recorded in llm_args (the snapshot is hybrid). Same submission as the recorded telecom row; the ledger scope already allows airline/retail.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "qwen3-max-2026-01-23",
    "bench": "tau2-airline",
    "score": 0.69,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/submission.json",
    "by": "lab",
    "date": "2026-01-26",
    "note": "Qwen's own submission (submitting_organization 'Qwen', submission_date 2026-01-26), model_name 'Qwen3-Max-Thinking'; results.airline.pass_1 = 69.0 (rounded to 4 dp; = 138/200 = 50 tasks x 4 trials). methodology: tau2_bench_version v0.1.3, user_simulator gpt-4.1-2025-04-14, modified_prompts false, omitted_questions false, 'Evaluation conducted using standard tau2-bench protocol'. Checkpoint: the trajectory file committed with the submission at 70e700c (web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/trajectories/airline_llm_agent_qwen3-max-2026-01-23_user_simulator_gpt-4.1-2025-04-14.json, removed from git in 20062c7) records agent llm 'qwen3-max-2026-01-23', temperature 0.0, 4 trials, max_steps 200; re-computed pass rate 0.69000. Thinking flag not recorded in llm_args (the snapshot is hybrid). Same submission as the recorded telecom row; the ledger scope already allows airline/retail.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "tau2-retail",
    "score": 0.722,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2024_09_23/submission.json",
    "by": "lab",
    "date": "2025-09-23",
    "note": "Submission dir qwen3-max_qwen_2024_09_23 (submitting_organization 'Alibaba', model_name 'Qwen3-Max', model_release.release_date 2025-09-23, reference 'Qwen3-Max: Just Scale it' blog); results.retail.pass_1 = 72.2. methodology: tau2_bench_version 0.1.3, user_simulator null, verification 'Unverified submission - no trajectory data available, unknown evaluation methodology'; notes 'Only average results are in the blog post. Evaluation limited to Pass^1, Pass^2 and Pass^4 scores only'. The submission's own submission_date/evaluation_date say 2024-09-23 (a typo for 2025: the model was released 2025-09-23); date set to the release date. Checkpoint: Qwen3-Max (Instruct) official release of 2025-09-23 = dated API snapshot qwen3-max-2025-09-23 (non-thinking). CAUTION: committed by Sierra staff (957599b / 9df0df8) from Qwen material; where the per-domain values come from is not stated. Same submission as the recorded telecom row.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "tau2-airline",
    "score": 0.595,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-max_qwen_2024_09_23/submission.json",
    "by": "lab",
    "date": "2025-09-23",
    "note": "Submission dir qwen3-max_qwen_2024_09_23 (submitting_organization 'Alibaba', model_name 'Qwen3-Max', model_release.release_date 2025-09-23, reference 'Qwen3-Max: Just Scale it' blog); results.airline.pass_1 = 59.5. methodology: tau2_bench_version 0.1.3, user_simulator null, verification 'Unverified submission - no trajectory data available, unknown evaluation methodology'; notes 'Only average results are in the blog post. Evaluation limited to Pass^1, Pass^2 and Pass^4 scores only'. The submission's own submission_date/evaluation_date say 2024-09-23 (a typo for 2025: the model was released 2025-09-23); date set to the release date. Checkpoint: Qwen3-Max (Instruct) official release of 2025-09-23 = dated API snapshot qwen3-max-2025-09-23 (non-thinking). CAUTION: committed by Sierra staff (957599b / 9df0df8) from Qwen material; where the per-domain values come from is not stated. Same submission as the recorded telecom row.",
    "source_id": "tau2-bench-leaderboard-qwen"
  },
  {
    "model": "mistral-large-2",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.955,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'mistralai/mistral-large-2411': factual consistency rate 95.5% (hallucination rate 4.5%), answer rate 99.9%, average summary length 85.0 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Identity: README API details 'Mistral Large: Accessed via Mistral AI's API using the model `mistral-large-2411`' = the Mistral-Large-Instruct-2411 checkpoint (catalogue mistral-large-2). Not recorded in the earlier import (missed mapping).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.918,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'google/gemini-3.1-flash-lite-preview': factual consistency rate 91.8% (hallucination rate 8.2%), answer rate 99.6%, average summary length 62.6 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Identity: Gemini API model id gemini-3.1-flash-lite-preview (row added in commit 83e10c4, 2026-03-04); mapped like the recorded 'google/gemini-3.1-pro-preview' -> gemini-3-1-pro row (the catalogue keeps no separate preview id for 3.1 Flash-Lite).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.914,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'deepseek-ai/DeepSeek-V4-Pro': factual consistency rate 91.4% (hallucination rate 8.6%), answer rate 97.2%, average summary length 153.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Identity: the row names the HF repo deepseek-ai/DeepSeek-V4-Pro (the April 2026 'DeepSeek-V4 Preview' Pro weights) and was added in commit 25a9e0a (2026-05-11, 'gpt 5.5, deepseek v4 pro, kimi k2.6'), before the 0813 GA (catalogue deepseek-v4-pro, HF DeepSeek-V4-Pro-0813) took over the API id on 2026-08-13. The README's API section has no DeepSeek V4 line.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "minimax-m2-1",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.882,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'MiniMaxAI/minimax-m2p1': factual consistency rate 88.2% (hallucination rate 11.8%), answer rate 98.5%, average summary length 106.9 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Identity: MiniMax-M2.1 (HF MiniMaxAI/MiniMax-M2.1); mapped like the recorded 'MiniMaxAI/minimax-m2p5' -> minimax-m2-5 and 'minimax-m2p7' -> minimax-m2-7 rows. minimax-m2-1 was added to the catalogue after the earlier import.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.933,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-2.0-flash-lite-001'; column 'Model Avg' = 0.933; single-turn 0.96, multi-turn 0.91; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Stable versioned Gemini API id (Vendor 'Google'). Needs the proposed comparison-only model gemini-2-0-flash-lite-001 (part C).",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "palmyra-x-004",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.886,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'palmyra-x-004'; column 'Model Avg' = 0.886; single-turn 0.92, multi-turn 0.85; 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Vendor 'Writer'. Needs the proposed comparison-only model palmyra-x-004.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "palmyra-x-004",
    "bench": "bfcl-v3",
    "score": 0.5684,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'palmyra-x-004 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 56.84% (Non-Live AST 86.38%, Live 77.21%, Multi Turn 5.88%). v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://writer.com/engineering/actions-with-palmyra-x-004/ (Organization 'Writer', License 'Proprietary'). Apache-2.0 (repository LICENSE). Needs the proposed comparison-only model palmyra-x-004.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "palmyra-x-004",
    "bench": "bfcl-v4",
    "score": 0.2787,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'palmyra-x-004 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 27.87% (Non-Live AST 87.46%, Live 77.87%, Multi Turn 0.38%, Web Search 2.50%, Memory 13.12%). Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://writer.com/engineering/actions-with-palmyra-x-004/ (Writer, Proprietary). Low multi-turn/agentic scores but single-turn AST in line with its v3 entry, so not a format failure. Needs the proposed comparison-only model palmyra-x-004.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "grok-4-1-fast-non-reasoning",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.822,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, last updated 22 September 2026), row 'xai-org/grok-4-1-fast-non-reasoning': factual consistency rate 82.2% (hallucination rate 17.8%), answer rate 98.5%, average summary length 87.5 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Separate API model from the recorded 'xai-org/grok-4-1-fast-reasoning' row (grok-4-1-fast-reasoning). Needs the proposed model grok-4-1-fast-non-reasoning (part C).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemini-2-5-pro-preview-03-25",
    "bench": "aider-polyglot",
    "score": 0.729,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-12",
    "note": "Published entry 'Gemini 2.5 Pro Preview 03-25'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=diff-fenced; Aider version=0.81.3.dev; evaluator commit=0282574; command=aider --model gemini/gemini-2.5-pro-preview-03-25; run=2025-04-12-04-55-50--gemini-25-pro-diff-fenced. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Run dated 2025-04-12, before Google repointed gemini-2.5-pro-preview-03-25 to the 05-06 model (2025-05-06), so it is the 03-25 preview. Needs the proposed comparison-only model gemini-2-5-pro-preview-03-25 (part E).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "livebench-20240726",
    "score": 0.5987,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'claude-3-5-sonnet-20240620'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 59.87. Category means: math 53.7, coding 60.8, reasoning 58.7, language 56.9, instruction_following 72.3, data_analysis 56.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "livebench-20240726",
    "score": 0.3586,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'claude-3-haiku-20240307'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 35.86. Category means: math 25.7, coding 24.5, reasoning 29.3, language 30.1, instruction_following 64.0, data_analysis 41.5. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "livebench-20240726",
    "score": 0.5056,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'claude-3-opus-20240229'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 50.56. Category means: math 46.5, coding 38.6, reasoning 41.3, language 51.7, instruction_following 70.9, data_analysis 54.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "livebench-20240726",
    "score": 0.3872,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'claude-3-sonnet-20240229'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 38.72. Category means: math 29.6, coding 26.4, reasoning 28.7, language 38.1, instruction_following 65.0, data_analysis 44.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "command-r-03-2024",
    "bench": "livebench-20240726",
    "score": 0.2683,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'command-r'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 26.83. Category means: math 16.9, coding 15.3, reasoning 25.3, language 14.6, instruction_following 57.2, data_analysis 31.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: API alias 'command-r'; model_answer @bc43a286 shows every answer to this question set was generated 2024-06-04..2024-07-22, before Command R 08-2024 existed (listed separately as 'command-r-08-2024'), so it is Command R 03-2024.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "livebench-20240726",
    "score": 0.3217,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'command-r-plus'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 32.17. Category means: math 24.9, coding 19.5, reasoning 28.7, language 23.9, instruction_following 71.5, data_analysis 24.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: API alias 'command-r-plus'; answers generated 2024-06-04..2024-07-22 (model_answer @bc43a286), before Command R+ 08-2024 existed (listed separately), so Command R+ 04-2024.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "command-r-08-2024",
    "bench": "livebench-20240726",
    "score": 0.3083,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'command-r-08-2024'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 30.83. Category means: math 26.3, coding 17.9, reasoning 26.7, language 17.0, instruction_following 65.7, data_analysis 31.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "livebench-20240726",
    "score": 0.345,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'command-r-plus-08-2024'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 34.50. Category means: math 26.4, coding 19.5, reasoning 30.7, language 31.0, instruction_following 63.6, data_analysis 35.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "deepseek-coder-v2-lite-instruct",
    "bench": "livebench-20240726",
    "score": 0.2953,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'deepseek-coder-v2-lite-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 29.53. Category means: math 34.4, coding 24.7, reasoning 26.0, language 10.6, instruction_following 48.3, data_analysis 33.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "deepseek-v2-lite-chat",
    "bench": "livebench-20240726",
    "score": 0.1774,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'deepseek-v2-lite-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 17.74. Category means: math 14.1, coding 7.1, reasoning 16.0, language 9.2, instruction_following 41.8, data_analysis 18.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "livebench-20240726",
    "score": 0.1878,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemma-1.1-7b-it'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 18.78. Category means: math 15.2, coding 9.6, reasoning 14.7, language 10.6, instruction_following 44.3, data_analysis 18.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemma-2-27b",
    "bench": "livebench-20240726",
    "score": 0.4126,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemma-2-27b-it'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 41.26. Category means: math 36.2, coding 35.9, reasoning 32.0, language 32.4, instruction_following 67.4, data_analysis 43.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemma-2-9b",
    "bench": "livebench-20240726",
    "score": 0.3134,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemma-2-9b-it'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 31.34. Category means: math 24.0, coding 22.5, reasoning 17.3, language 27.6, instruction_following 61.5, data_analysis 35.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "livebench-20240726",
    "score": 0.3454,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-3.5-turbo-0125'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 34.54. Category means: math 26.9, coding 27.7, reasoning 26.7, language 24.2, instruction_following 60.5, data_analysis 41.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "livebench-20240726",
    "score": 0.3407,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-3.5-turbo-1106'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 34.07. Category means: math 28.5, coding 26.1, reasoning 28.0, language 28.6, instruction_following 51.5, data_analysis 41.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "livebench-20240726",
    "score": 0.489,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4-0125-preview'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 48.90. Category means: math 42.7, coding 41.8, reasoning 47.3, language 43.6, instruction_following 63.9, data_analysis 54.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4-0613",
    "bench": "livebench-20240726",
    "score": 0.456,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4-0613'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 45.60. Category means: math 36.2, coding 37.3, reasoning 34.7, language 49.6, instruction_following 71.8, data_analysis 44.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "livebench-20240726",
    "score": 0.5197,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4-1106-preview'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 51.97. Category means: math 47.6, coding 47.2, reasoning 48.0, language 48.4, instruction_following 69.4, data_analysis 51.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "livebench-20240726",
    "score": 0.5288,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4-turbo-2024-04-09'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 52.88. Category means: math 49.0, coding 49.0, reasoning 51.3, language 45.3, instruction_following 71.4, data_analysis 51.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "livebench-20240726",
    "score": 0.5463,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4o-2024-05-13'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 54.63. Category means: math 49.9, coding 49.4, reasoning 50.0, language 53.9, instruction_following 72.2, data_analysis 52.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "livebench-20240726",
    "score": 0.5671,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4o-2024-08-06'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 56.71. Category means: math 52.3, coding 51.4, reasoning 54.7, language 54.4, instruction_following 74.6, data_analysis 52.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "livebench-20240726",
    "score": 0.4426,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gpt-4o-mini-2024-07-18'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 44.26. Category means: math 41.6, coding 43.2, reasoning 35.3, language 35.3, instruction_following 65.7, data_analysis 44.5. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "livebench-20240726",
    "score": 0.1163,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'llama-2-7b-chat-hf'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 11.63. Category means: math 4.8, coding 1.3, reasoning 12.0, language 6.9, instruction_following 44.9, data_analysis 0.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mathstral-7b-v0-1",
    "bench": "livebench-20240726",
    "score": 0.2448,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mathstral-7b-v0.1'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 24.48. Category means: math 17.8, coding 14.5, reasoning 18.0, language 15.4, instruction_following 53.2, data_analysis 27.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-70b",
    "bench": "livebench-20240726",
    "score": 0.3773,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'meta-llama-3-70b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 37.73. Category means: math 32.3, coding 22.0, reasoning 30.7, language 34.1, instruction_following 63.5, data_analysis 43.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "livebench-20240726",
    "score": 0.2762,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'meta-llama-3-8b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 27.62. Category means: math 19.7, coding 19.8, reasoning 24.0, language 18.7, instruction_following 57.1, data_analysis 26.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "livebench-20240726",
    "score": 0.5425,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'meta-llama-3.1-405b-instruct-turbo'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 54.25. Category means: math 46.5, coding 43.8, reasoning 53.3, language 49.8, instruction_following 78.5, data_analysis 53.5. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: served as Together AI 'Meta-Llama-3.1-405B-Instruct-Turbo' (Together's FP8 endpoint of the same weights).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "livebench-20240726",
    "score": 0.4844,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'meta-llama-3.1-70b-instruct-turbo'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 48.44. Category means: math 45.6, coding 32.7, reasoning 40.7, language 42.4, instruction_following 79.1, data_analysis 50.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: served as Together AI 'Meta-Llama-3.1-70B-Instruct-Turbo' (FP8 endpoint of the same weights).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "livebench-20240726",
    "score": 0.2803,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'meta-llama-3.1-8b-instruct-turbo'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 28.03. Category means: math 24.4, coding 19.7, reasoning 15.3, language 20.0, instruction_following 56.5, data_analysis 32.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: served as Together AI 'Meta-Llama-3.1-8B-Instruct-Turbo' (FP8 endpoint of the same weights).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "livebench-20240726",
    "score": 0.2005,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mistral-7b-instruct-v0.2'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 20.05. Category means: math 17.1, coding 13.9, reasoning 14.0, language 9.1, instruction_following 51.7, data_analysis 14.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-3",
    "bench": "livebench-20240726",
    "score": 0.2125,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mistral-7b-instruct-v0.3'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 21.25. Category means: math 14.6, coding 11.0, reasoning 16.0, language 11.9, instruction_following 52.4, data_analysis 21.8. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-large-2402",
    "bench": "livebench-20240726",
    "score": 0.3918,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mistral-large-2402'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 39.18. Category means: math 32.2, coding 27.4, reasoning 36.0, language 28.7, instruction_following 68.2, data_analysis 42.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-large-2407",
    "bench": "livebench-20240726",
    "score": 0.4797,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mistral-large-2407'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 47.97. Category means: math 40.5, coding 47.1, reasoning 42.0, language 39.8, instruction_following 71.8, data_analysis 46.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "livebench-20240726",
    "score": 0.2917,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'open-mistral-nemo'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 29.17. Category means: math 21.7, coding 28.7, reasoning 25.3, language 14.1, instruction_following 51.8, data_analysis 33.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: Mistral API name 'open-mistral-nemo' (Mistral NeMo 12B, 2407 is its only version); answers generated 2024-07-24..25.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "livebench-20240726",
    "score": 0.2413,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'openhermes-2.5-mistral-7b'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 24.13. Category means: math 20.4, coding 13.3, reasoning 20.0, language 11.4, instruction_following 52.8, data_analysis 26.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "livebench-20240726",
    "score": 0.303,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'phi-3-medium-128k-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 30.30. Category means: math 25.6, coding 21.1, reasoning 34.0, language 12.8, instruction_following 56.2, data_analysis 32.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "livebench-20240726",
    "score": 0.3122,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'phi-3-medium-4k-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 31.22. Category means: math 31.4, coding 20.5, reasoning 36.7, language 13.9, instruction_following 53.3, data_analysis 31.6. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "phi-3-5-mini-instruct",
    "bench": "livebench-20240726",
    "score": 0.283,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'phi-3.5-mini-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 28.30. Category means: math 22.2, coding 15.9, reasoning 33.3, language 9.7, instruction_following 58.3, data_analysis 30.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "phi-3-5-moe-instruct",
    "bench": "livebench-20240726",
    "score": 0.3516,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'phi-3.5-moe-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 35.16. Category means: math 33.3, coding 21.7, reasoning 38.7, language 17.1, instruction_following 59.7, data_analysis 40.5. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "livebench-20240726",
    "score": 0.2978,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-110b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 29.78. Category means: math 26.3, coding 21.8, reasoning 30.7, language 13.2, instruction_following 55.3, data_analysis 31.5. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-4b-chat",
    "bench": "livebench-20240726",
    "score": 0.1128,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-4b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 11.28. Category means: math 9.9, coding 4.5, reasoning 10.7, language 5.8, instruction_following 27.8, data_analysis 9.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "livebench-20240726",
    "score": 0.2926,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-72b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 29.26. Category means: math 26.8, coding 22.8, reasoning 23.3, language 11.4, instruction_following 58.2, data_analysis 33.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "livebench-20240726",
    "score": 0.1798,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-7b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 17.98. Category means: math 15.3, coding 9.4, reasoning 16.0, language 6.2, instruction_following 44.1, data_analysis 16.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen2-0-5b",
    "bench": "livebench-20240726",
    "score": 0.0768,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2-0.5b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 7.68. Category means: math 7.3, coding 1.3, reasoning 6.0, language 2.8, instruction_following 26.6, data_analysis 2.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "livebench-20240726",
    "score": 0.1035,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2-1.5b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 10.35. Category means: math 9.9, coding 5.2, reasoning 8.0, language 3.0, instruction_following 25.9, data_analysis 10.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen2-72b",
    "bench": "livebench-20240726",
    "score": 0.4015,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2-72b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 40.15. Category means: math 43.4, coding 32.4, reasoning 41.3, language 29.2, instruction_following 68.3, data_analysis 26.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen2-7b",
    "bench": "livebench-20240726",
    "score": 0.2658,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2-7b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 26.58. Category means: math 26.9, coding 28.9, reasoning 20.0, language 10.2, instruction_following 44.7, data_analysis 28.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "livebench-20240726",
    "score": 0.166,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'starling-lm-7b-beta'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 16.60. Category means: math 14.9, coding 18.5, reasoning 18.7, language 7.3, instruction_following 38.3, data_analysis 2.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "livebench-20240726",
    "score": 0.1257,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'vicuna-7b-v1.5'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 12.57. Category means: math 7.1, coding 1.9, reasoning 12.7, language 8.7, instruction_following 41.8, data_analysis 3.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "livebench-20240726",
    "score": 0.186,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'zephyr-7b-alpha'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 18.60. Category means: math 10.0, coding 12.3, reasoning 12.0, language 7.2, instruction_following 52.8, data_analysis 17.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "livebench-20240726",
    "score": 0.1672,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'zephyr-7b-beta'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 16.72. Category means: math 11.2, coding 8.1, reasoning 12.7, language 4.3, instruction_following 48.3, data_analysis 15.7. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "bigcodebench-complete",
    "score": 0.602,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'GPT-4o' / checkpoint '2024-05-13', column 'Greedy Decoding · Original' = 0.602 (calibrated 0.611). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: results tables print checkpoint 2024-05-13 and §M prose says 'GPT-4o-2024-05-13'; NOTE Table 4 (artifacts) lists 'GPT-4o | gpt-4-turbo-2024-04-09', an evident row shift (owner review flag).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "bigcodebench-instruct",
    "score": 0.499,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'GPT-4o' / checkpoint '2024-05-13', column 'Greedy Decoding · Original' = 0.499 (calibrated 0.511). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: results tables print checkpoint 2024-05-13 and §M prose says 'GPT-4o-2024-05-13'; NOTE Table 4 (artifacts) lists 'GPT-4o | gpt-4-turbo-2024-04-09', an evident row shift (owner review flag).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-4-0613",
    "bench": "bigcodebench-complete",
    "score": 0.484,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'GPT-4' / checkpoint '0613', column 'Greedy Decoding · Original' = 0.484 (calibrated 0.572). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: gpt-4-0613.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-4-0613",
    "bench": "bigcodebench-instruct",
    "score": 0.459,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'GPT-4' / checkpoint '0613', column 'Greedy Decoding · Original' = 0.459 (calibrated 0.46). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: gpt-4-0613.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "bigcodebench-complete",
    "score": 0.506,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'GPT-3.5-Turbo' / checkpoint '0125', column 'Greedy Decoding · Original' = 0.506 (calibrated 0.506). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: gpt-3.5-turbo-0125.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "bigcodebench-instruct",
    "score": 0.382,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'GPT-3.5-Turbo' / checkpoint '0125', column 'Greedy Decoding · Original' = 0.382 (calibrated 0.391). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: gpt-3.5-turbo-0125.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "bigcodebench-complete",
    "score": 0.568,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Claude-3' / checkpoint 'Opus', column 'Greedy Decoding · Original' = 0.568 (calibrated 0.574). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-opus-20240229.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "bigcodebench-instruct",
    "score": 0.452,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Claude-3' / checkpoint 'Opus', column 'Greedy Decoding · Original' = 0.452 (calibrated 0.455). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-opus-20240229.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "bigcodebench-complete",
    "score": 0.534,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Claude-3' / checkpoint 'Sonnet', column 'Greedy Decoding · Original' = 0.534 (calibrated 0.538). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-sonnet-20240229.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "bigcodebench-instruct",
    "score": 0.425,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Claude-3' / checkpoint 'Sonnet', column 'Greedy Decoding · Original' = 0.425 (calibrated 0.427). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-sonnet-20240229.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "bigcodebench-complete",
    "score": 0.486,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Claude-3' / checkpoint 'Haiku', column 'Greedy Decoding · Original' = 0.486 (calibrated 0.501). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-haiku-20240307.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "bigcodebench-instruct",
    "score": 0.392,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Claude-3' / checkpoint 'Haiku', column 'Greedy Decoding · Original' = 0.392 (calibrated 0.394). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: claude-3-haiku-20240307.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mistral-large-2402",
    "bench": "bigcodebench-complete",
    "score": 0.379,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Mistral' / checkpoint 'Large', column 'Greedy Decoding · Original' = 0.379 (calibrated 0.383). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: mistral-large-2402.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mistral-large-2402",
    "bench": "bigcodebench-instruct",
    "score": 0.296,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Mistral' / checkpoint 'Large', column 'Greedy Decoding · Original' = 0.296 (calibrated 0.3). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: mistral-large-2402.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.511,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-instruct' / checkpoint '33B', column 'Greedy Decoding · Original' = 0.511 (calibrated 0.511). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-33b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-33b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.418,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'DeepSeek-Coder-instruct' / checkpoint '33B', column 'Greedy Decoding · Original' = 0.418 (calibrated 0.42). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-33b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.432,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-instruct' / checkpoint '6.7B', column 'Greedy Decoding · Original' = 0.432 (calibrated 0.438). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-6-7b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.353,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'DeepSeek-Coder-instruct' / checkpoint '6.7B', column 'Greedy Decoding · Original' = 0.353 (calibrated 0.355). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.293,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-instruct' / checkpoint '1.3B', column 'Greedy Decoding · Original' = 0.293 (calibrated 0.296). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-1-3b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.227,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'DeepSeek-Coder-instruct' / checkpoint '1.3B', column 'Greedy Decoding · Original' = 0.227 (calibrated 0.228). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.489,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama-instruct' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.489 (calibrated 0.496). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-70b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.405,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeLlama-instruct' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.405 (calibrated 0.407). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-70b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.351,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama-instruct' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.351 (calibrated 0.356). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-34b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.289,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeLlama-instruct' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.289 (calibrated 0.29). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-34b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.297,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama-instruct' / checkpoint '13B', column 'Greedy Decoding · Original' = 0.297 (calibrated 0.317). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-13b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-13b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.282,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeLlama-instruct' / checkpoint '13B', column 'Greedy Decoding · Original' = 0.282 (calibrated 0.285). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-13b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.254,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama-instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.254 (calibrated 0.257). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-7b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-7b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.218,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeLlama-instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.218 (calibrated 0.219). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-7b-Instruct-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-70b",
    "bench": "bigcodebench-complete",
    "score": 0.539,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Llama3-instruct' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.539 (calibrated 0.545). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-70b",
    "bench": "bigcodebench-instruct",
    "score": 0.432,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Llama3-instruct' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.432 (calibrated 0.436). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-8b",
    "bench": "bigcodebench-complete",
    "score": 0.368,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Llama3-instruct' / checkpoint '8B', column 'Greedy Decoding · Original' = 0.368 (calibrated 0.369). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-8b",
    "bench": "bigcodebench-instruct",
    "score": 0.317,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Llama3-instruct' / checkpoint '8B', column 'Greedy Decoding · Original' = 0.317 (calibrated 0.319). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen2-72b",
    "bench": "bigcodebench-complete",
    "score": 0.539,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen2-Instruct' / checkpoint '72B', column 'Greedy Decoding · Original' = 0.539 (calibrated 0.54). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen2-72B-Instruct. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen2-72b",
    "bench": "bigcodebench-instruct",
    "score": 0.382,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen2-Instruct' / checkpoint '72B', column 'Greedy Decoding · Original' = 0.382 (calibrated 0.385). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen2-72B-Instruct. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen2-7b",
    "bench": "bigcodebench-complete",
    "score": 0.407,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen2-Instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.407 (calibrated 0.421). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen2-7B-Instruct. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen2-7b",
    "bench": "bigcodebench-instruct",
    "score": 0.28,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen2-Instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.28 (calibrated 0.291). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen2-7B-Instruct. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.437,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeQwen1.5-Chat' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.437 (calibrated 0.443). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/CodeQwen1.5-7B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.393,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeQwen1.5-Chat' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.393 (calibrated 0.396). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/CodeQwen1.5-7B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.432,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen1.5-Chat' / checkpoint '110B', column 'Greedy Decoding · Original' = 0.432 (calibrated 0.444). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-110B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.336,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen1.5-Chat' / checkpoint '110B', column 'Greedy Decoding · Original' = 0.336 (calibrated 0.35). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-110B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.398,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen1.5-Chat' / checkpoint '72B', column 'Greedy Decoding · Original' = 0.398 (calibrated 0.403). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-72B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.326,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen1.5-Chat' / checkpoint '72B', column 'Greedy Decoding · Original' = 0.326 (calibrated 0.332). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-72B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.41,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen1.5-Chat' / checkpoint '32B', column 'Greedy Decoding · Original' = 0.41 (calibrated 0.42). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-32B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.318,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen1.5-Chat' / checkpoint '32B', column 'Greedy Decoding · Original' = 0.318 (calibrated 0.323). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/Qwen1.5-32B-Chat. Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.433,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5-Chat' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.433 (calibrated 0.428). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-34B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.335,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Yi-1.5-Chat' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.335 (calibrated 0.339). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-34B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.422,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5-Chat' / checkpoint '9B', column 'Greedy Decoding · Original' = 0.422 (calibrated 0.424). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-9B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-9b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.343,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Yi-1.5-Chat' / checkpoint '9B', column 'Greedy Decoding · Original' = 0.343 (calibrated 0.345). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-9B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "bigcodebench-complete",
    "score": 0.338,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5-Chat' / checkpoint '6B', column 'Greedy Decoding · Original' = 0.338 (calibrated 0.339). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-6B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-6b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.255,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Yi-1.5-Chat' / checkpoint '6B', column 'Greedy Decoding · Original' = 0.255 (calibrated 0.256). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-6B-Chat.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "bigcodebench-complete",
    "score": 0.496,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Mixtral-instruct (Table 7: 'Mixtral')' / checkpoint '8x22B', column 'Greedy Decoding · Original' = 0.496 (calibrated 0.502). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "bigcodebench-instruct",
    "score": 0.404,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Mixtral-instruct (Table 7: 'Mixtral')' / checkpoint '8x22B', column 'Greedy Decoding · Original' = 0.404 (calibrated 0.406). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "bigcodebench-complete",
    "score": 0.475,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Magicoder-S-DS' / checkpoint '6.7B', column 'Greedy Decoding · Original' = 0.475 (calibrated 0.476). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B. Marked lab: co-author Jiawei Liu (UIUC) is a Magicoder author.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "magicoder-s-ds-6-7b",
    "bench": "bigcodebench-instruct",
    "score": 0.36,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Magicoder-S-DS' / checkpoint '6.7B', column 'Greedy Decoding · Original' = 0.36 (calibrated 0.362). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B. Marked lab: co-author Jiawei Liu (UIUC) is a Magicoder author.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "bigcodebench-complete",
    "score": 0.383,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeGemma-instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.383 (calibrated 0.393). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/google/codegemma-7b-it.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codegemma-7b-it",
    "bench": "bigcodebench-instruct",
    "score": 0.321,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'CodeGemma-instruct' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.321 (calibrated 0.323). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/google/codegemma-7b-it.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "bigcodebench-complete",
    "score": 0.421,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code-Instruct' / checkpoint '20B', column 'Greedy Decoding · Original' = 0.421 (calibrated 0.421). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-20b-code-instruct (HF API now 307-redirects to ibm-granite/granite-20b-code-instruct-8k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-20b-code-instruct-8k",
    "bench": "bigcodebench-instruct",
    "score": 0.337,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Granite-Code-Instruct' / checkpoint '20B', column 'Greedy Decoding · Original' = 0.337 (calibrated 0.34). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-20b-code-instruct (HF API now 307-redirects to ibm-granite/granite-20b-code-instruct-8k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-8b-code-instruct-4k",
    "bench": "bigcodebench-complete",
    "score": 0.392,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code-Instruct' / checkpoint '8B', column 'Greedy Decoding · Original' = 0.392 (calibrated 0.397). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-8b-code-instruct (HF API now 307-redirects to ibm-granite/granite-8b-code-instruct-4k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.' Not in Table 7 ('We do not include the results of Granite-Code-Instruct 8B & 3B ...').",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-70b",
    "bench": "bigcodebench-complete",
    "score": 0.443,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama (base)' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.443 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-70b-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-34b",
    "bench": "bigcodebench-complete",
    "score": 0.371,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama (base)' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.371 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-34b-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-13b",
    "bench": "bigcodebench-complete",
    "score": 0.32,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama (base)' / checkpoint '13B', column 'Greedy Decoding · Original' = 0.32 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-13b-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codellama-7b",
    "bench": "bigcodebench-complete",
    "score": 0.287,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeLlama (base)' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.287 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/codellama/CodeLlama-7b-hf.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-33b-base",
    "bench": "bigcodebench-complete",
    "score": 0.466,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-base' / checkpoint '33B', column 'Greedy Decoding · Original' = 0.466 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-33b-base.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-6-7b-base",
    "bench": "bigcodebench-complete",
    "score": 0.418,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-base' / checkpoint '6.7B', column 'Greedy Decoding · Original' = 0.418 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-6.7b-base.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "deepseek-coder-1-3b-base",
    "bench": "bigcodebench-complete",
    "score": 0.223,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'DeepSeek-Coder-base' / checkpoint '1.3B', column 'Greedy Decoding · Original' = 0.223 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/deepseek-ai/deepseek-coder-1.3b-base.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codegemma-7b",
    "bench": "bigcodebench-complete",
    "score": 0.373,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeGemma (base)' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.373 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/google/codegemma-7b.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codegemma-2b",
    "bench": "bigcodebench-complete",
    "score": 0.239,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeGemma (base)' / checkpoint '2B', column 'Greedy Decoding · Original' = 0.239 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/google/codegemma-2b.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "starcoder2-15b",
    "bench": "bigcodebench-complete",
    "score": 0.383,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'StarCoder2' / checkpoint '15B', column 'Greedy Decoding · Original' = 0.383 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/bigcode/starcoder2-15b. StarCoder2 is a BigCode model and several authors (Harm de Vries, Leandro von Werra, Qian Liu, Jenny Chim) are StarCoder2 authors, so by: lab. Table 6 caption: trailing newlines stripped for StarCoder2.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "starcoder2-7b",
    "bench": "bigcodebench-complete",
    "score": 0.285,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'StarCoder2' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.285 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/bigcode/starcoder2-7b. StarCoder2 is a BigCode model and several authors are StarCoder2 authors, so by: lab. Table 6 caption: trailing newlines stripped for StarCoder2.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "starcoder2-3b",
    "bench": "bigcodebench-complete",
    "score": 0.214,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'StarCoder2' / checkpoint '3B', column 'Greedy Decoding · Original' = 0.214 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/bigcode/starcoder2-3b. StarCoder2 is a BigCode model and several authors are StarCoder2 authors, so by: lab. Table 6 caption: trailing newlines stripped for StarCoder2.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-20b-code-base-8k",
    "bench": "bigcodebench-complete",
    "score": 0.258,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code (base)' / checkpoint '20B', column 'Greedy Decoding · Original' = 0.258 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-20b-code-base (HF API now 307-redirects to ibm-granite/granite-20b-code-base-8k, the catalogue repo). Table 6 caption: trailing newlines stripped for Granite-Code.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-8b-code-base-4k",
    "bench": "bigcodebench-complete",
    "score": 0.356,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code (base)' / checkpoint '8B', column 'Greedy Decoding · Original' = 0.356 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-8b-code-base (HF API now 307-redirects to ibm-granite/granite-8b-code-base-4k, the catalogue repo). Table 6 caption: trailing newlines stripped for Granite-Code.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-3-5-sonnet-20241022",
    "bench": "ifeval-prompt-strict",
    "score": 0.8558,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'claude-3-5-sonnet-20241022' → results 'Prompt-level-strict-accuracy' = 85.5823 (inst-level strict 89.69, prompt-level loose 89.65). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'claude-3-5-sonnet-20241022' (OpenCompass ClaudeAllesAPIN gateway), max_out_len 8192.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "ifeval-prompt-strict",
    "score": 0.8725,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'claude-3-7-sonnet-20250219' → results 'Prompt-level-strict-accuracy' = 87.2458 (inst-level strict 91.37, prompt-level loose 90.57). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'claude-3-7-sonnet-20250219' via opencompass.models.ClaudeSDK; no thinking/budget parameter in the config, so standard (non-extended-thinking) mode — the only setting this source reports.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "ifeval-prompt-strict",
    "score": 0.7763,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'deepseek-v2_5-turbomind' → results 'Prompt-level-strict-accuracy' = 77.6340 (inst-level strict 84.53, prompt-level loose 81.70). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'deepseek-ai/DeepSeek-V2.5' run locally with LMDeploy (TurboMindModelwithChatTemplate, pytorch backend, tp 8), greedy (temperature 1e-6, top_k 1).",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "ifeval-prompt-strict",
    "score": 0.793,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'deepseek-v2_5-1210-turbomind' → results 'Prompt-level-strict-accuracy' = 79.2976 (inst-level strict 85.49, prompt-level loose 82.81). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'deepseek-ai/DeepSeek-V2.5-1210' run locally with LMDeploy, greedy.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v3",
    "bench": "ifeval-prompt-strict",
    "score": 0.8577,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'deepseek-chat-v3' → results 'Prompt-level-strict-accuracy' = 85.7671 (inst-level strict 90.29, prompt-level loose 89.46). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: abbr 'deepseek-chat-v3' served from a Volcengine Ark endpoint 'ep-m-20250304164737-crpx6' (created 2025-03-04, before DeepSeek-V3-0324 was released on 2025-03-24) and listed separately from 'deepseek-chat-v3-0324' in the same file, so it is the original DeepSeek-V3 (Dec '24); temperature 1e-4.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "ifeval-prompt-strict",
    "score": 0.8189,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'deepseek-chat-v3-0324' → results 'Prompt-level-strict-accuracy' = 81.8854 (inst-level strict 87.41, prompt-level loose 86.51). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'DeepSeek-V3-0324' on a self-hosted OpenAI-compatible server; temperature 1e-4.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-r1",
    "bench": "ifeval-prompt-strict",
    "score": 0.8336,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (opencompass/CompassAcademic-Leaderboard-Full-Version @73af091f, apache-2.0), committed file data/hf-academic-predictions.json (last changed in commit 7ff51fcd, 2025-03-31), key IFEval → 'deepseek-chat-r1' → results 'Prompt-level-strict-accuracy' = 83.3641 (inst-level strict 88.97, prompt-level loose 87.06). OpenCompass's own run: IFEval 541 prompts, zero-shot (ZeroRetriever), IFEvaluator; the same entry stores all 541 predictions. Identity: cfg path 'DeepSeek-R1-BF16' on a self-hosted server, reasoning content stripped before scoring (extract_non_reasoning_content); file committed 2025-03-31, before R1-0528 existed, so original DeepSeek-R1 (Jan '25); max 32768 output tokens.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v2-5",
    "bench": "simple-evals-drop-f1",
    "score": 0.8138,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (@73af091f, apache-2.0), data/hf-academic-predictions.json, key drop → 'deepseek-v2_5-turbomind' → results 'accuracy' = 81.3758 (OpenCompass's DropOpenAIEvaluator reports the simple-evals F1 as 'accuracy'; the other entries print f1 = accuracy/100). Same protocol as simple-evals DROP: the stored prompt has the simple-evals wording with 3 worked examples and 'Think step by step, then write a line of the form \"Answer: $ANSWER\"', DropOpenAIEvaluator (simple-evals F1), 9,536 dev questions; OpenCompass's own run. Identity: cfg path 'deepseek-ai/DeepSeek-V2.5' run locally with LMDeploy (TurboMindModelwithChatTemplate, pytorch backend, tp 8), greedy (temperature 1e-6, top_k 1).",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "simple-evals-drop-f1",
    "score": 0.8908,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (@73af091f, apache-2.0), data/hf-academic-predictions.json, key drop → 'deepseek-v2_5-1210-turbomind' → results 'accuracy' = 89.0835 (f1 0.8908). Same protocol as simple-evals DROP: the stored prompt has the simple-evals wording with 3 worked examples and 'Think step by step, then write a line of the form \"Answer: $ANSWER\"', DropOpenAIEvaluator (simple-evals F1), 9,536 dev questions; OpenCompass's own run. Identity: cfg path 'deepseek-ai/DeepSeek-V2.5-1210' run locally with LMDeploy, greedy.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "deepseek-v3",
    "bench": "simple-evals-drop-f1",
    "score": 0.9212,
    "source": "https://huggingface.co/spaces/opencompass/CompassAcademic-Leaderboard-Full-Version/blob/73af091f28193d1e3e232851b940cadbb856556e/data/hf-academic-predictions.json",
    "by": "independent",
    "date": "2025-03-31",
    "note": "OpenCompass CompassAcademic leaderboard Space (@73af091f, apache-2.0), data/hf-academic-predictions.json, key drop → 'deepseek-chat-v3' → results 'accuracy' = 92.1246 (f1 0.9212). Same protocol as simple-evals DROP: the stored prompt has the simple-evals wording with 3 worked examples and 'Think step by step, then write a line of the form \"Answer: $ANSWER\"', DropOpenAIEvaluator (simple-evals F1), 9,536 dev questions; OpenCompass's own run. Identity: abbr 'deepseek-chat-v3' served from a Volcengine Ark endpoint 'ep-m-20250304164737-crpx6' (created 2025-03-04, before DeepSeek-V3-0324 was released on 2025-03-24) and listed separately from 'deepseek-chat-v3-0324' in the same file, so it is the original DeepSeek-V3 (Dec '24); temperature 1e-4.",
    "source_id": "opencompass-compassacademic-space-predictions"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "ruler-64k",
    "score": 0.928,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Jamba-1.5-mini' (claimed length 256K), column '64K' = 92.8: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/ai21labs/AI21-Jamba-1.5-Mini (HF API 307-redirects to ai21labs/AI21-Jamba-Mini-1.5, the catalogue repo).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "jamba-1-5-mini",
    "bench": "ruler-128k",
    "score": 0.9,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Jamba-1.5-mini' (claimed length 256K), column '128K' = 90.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/ai21labs/AI21-Jamba-1.5-Mini (HF API 307-redirects to ai21labs/AI21-Jamba-Mini-1.5, the catalogue repo).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "ruler-64k",
    "score": 0.87,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GPT-4-1106-preview' (claimed length 128K), column '64K' = 87.0: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links OpenAI API gpt-4-1106-preview.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "ruler-128k",
    "score": 0.812,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GPT-4-1106-preview' (claimed length 128K), column '128K' = 81.2: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links OpenAI API gpt-4-1106-preview.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "ruler-64k",
    "score": 0.884,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Llama3.1 (70B)' (claimed length 128K), column '64K' = 88.4: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct (now meta-llama/Llama-3.1-70B-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "ruler-128k",
    "score": 0.666,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Llama3.1 (70B)' (claimed length 128K), column '128K' = 66.6: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct (now meta-llama/Llama-3.1-70B-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-large-2",
    "bench": "ruler-64k",
    "score": 0.859,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Large-2411' (claimed length 128K), column '64K' = 85.9: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Large-Instruct-2411.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-large-2",
    "bench": "ruler-128k",
    "score": 0.481,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Large-2411' (claimed length 128K), column '128K' = 48.1: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Large-Instruct-2411.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "ruler-64k",
    "score": 0.854,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-plus-0824' (claimed length 128K), column '64K' = 85.4: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-plus-08-2024",
    "bench": "ruler-128k",
    "score": 0.646,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-plus-0824' (claimed length 128K), column '128K' = 64.6: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "ruler-64k",
    "score": 0.798,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Qwen2 (72B)' (claimed length 128K), column '64K' = 79.8: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/Qwen/Qwen2-72B-Instruct.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "qwen2-72b",
    "bench": "ruler-128k",
    "score": 0.537,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Qwen2 (72B)' (claimed length 128K), column '128K' = 53.7: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/Qwen/Qwen2-72B-Instruct.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "ruler-64k",
    "score": 0.843,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-plus' (claimed length 128K), column '64K' = 84.3: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-plus (the 04-2024 weights).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "ruler-128k",
    "score": 0.631,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-plus' (claimed length 128K), column '128K' = 63.1: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-plus (the 04-2024 weights).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-08-2024",
    "bench": "ruler-64k",
    "score": 0.866,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-0824' (claimed length 128K), column '64K' = 86.6: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-08-2024.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-08-2024",
    "bench": "ruler-128k",
    "score": 0.747,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R-0824' (claimed length 128K), column '128K' = 74.7: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-08-2024.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "ruler-64k",
    "score": 0.847,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Llama3.1 (8B)' (claimed length 128K), column '64K' = 84.7: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct (now meta-llama/Llama-3.1-8B-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "ruler-128k",
    "score": 0.77,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Llama3.1 (8B)' (claimed length 128K), column '128K' = 77.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct (now meta-llama/Llama-3.1-8B-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-03-2024",
    "bench": "ruler-64k",
    "score": 0.849,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R' (claimed length 128K), column '64K' = 84.9: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-v01.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "command-r-03-2024",
    "bench": "ruler-128k",
    "score": 0.76,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Command-R' (claimed length 128K), column '128K' = 76.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/CohereForAI/c4ai-command-r-v01.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "ruler-64k",
    "score": 0.788,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Large-2407' (claimed length 128K), column '64K' = 78.8: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Large-Instruct-2407.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-large-2407",
    "bench": "ruler-128k",
    "score": 0.237,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Large-2407' (claimed length 128K), column '128K' = 23.7: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Large-Instruct-2407.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "ruler-64k",
    "score": 0.847,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mixtral-8x22B' (claimed length 64K), column '64K' = 84.7: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mixtral-8x22B-instruct-v0.1. 128K column not used: beyond the model's claimed 64K context.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "ruler-64k",
    "score": 0.786,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Phi3-medium' (claimed length 128K), column '64K' = 78.6: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/microsoft/Phi-3-medium-128K-instruct.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "phi-3-medium-128k-instruct",
    "bench": "ruler-128k",
    "score": 0.461,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Phi3-medium' (claimed length 128K), column '128K' = 46.1: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/microsoft/Phi-3-medium-128K-instruct.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "ruler-64k",
    "score": 0.468,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Nemo' (claimed length 128K), column '64K' = 46.8: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Nemo-Instruct-2407.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "mistral-nemo-instruct-2407",
    "bench": "ruler-128k",
    "score": 0.19,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Mistral-Nemo' (claimed length 128K), column '128K' = 19.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Identity: row links huggingface.co/mistralai/Mistral-Nemo-Instruct-2407.",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.64,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o (0513)', column 'MMMU (EM)' = 0.640. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o (2024-05-13)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.551,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o (0513)', column 'MathVista (EM)' = 0.551. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o (2024-05-13)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.63,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o (0806)', column 'MMMU (EM)' = 0.630. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o (2024-08-06)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.567,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o (0806)', column 'MathVista (EM)' = 0.567. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o (2024-08-06)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.655,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.5 Sonnet', column 'MMMU (EM)' = 0.655. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3.5 Sonnet (20240620)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.575,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.5 Sonnet', column 'MathVista (EM)' = 0.575. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3.5 Sonnet (20240620)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.554,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4 Turbo', column 'MMMU (EM)' = 0.554. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4 Turbo (2024-04-09)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.493,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4 Turbo', column 'MathVista (EM)' = 0.493. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4 Turbo (2024-04-09)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.548,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o mini', column 'MMMU (EM)' = 0.548. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o mini (2024-07-18)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.475,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'GPT-4o mini', column 'MathVista (EM)' = 0.475. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'GPT-4o mini (2024-07-18)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.619,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Gemini 1.5 Pro(001)', column 'MMMU (EM)' = 0.619. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Gemini 1.5 Pro (001)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.576,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Gemini 1.5 Pro(001)', column 'MathVista (EM)' = 0.576. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Gemini 1.5 Pro (001)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.566,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Gemini 1.5 Flash (001)', column 'MMMU (EM)' = 0.566. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Gemini 1.5 Flash (001)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.512,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Gemini 1.5 Flash (001)', column 'MathVista (EM)' = 0.512. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Gemini 1.5 Flash (001)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.532,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Opus', column 'MMMU (EM)' = 0.532. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Opus (20240229)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.405,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Opus', column 'MathVista (EM)' = 0.405. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Opus (20240229)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.447,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Sonnet', column 'MMMU (EM)' = 0.447. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Sonnet (20240229)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.393,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Sonnet', column 'MathVista (EM)' = 0.393. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Sonnet (20240229)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "vhelm-mmmu-val-em",
    "score": 0.481,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Haiku', column 'MMMU (EM)' = 0.481. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Haiku (20240307)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "vhelm-mathvista-testmini-em",
    "score": 0.352,
    "source": "https://arxiv.org/html/2410.07112v2",
    "by": "independent",
    "date": "2024-10-24",
    "note": "VHELM paper (Lee et al., Stanford CRFM, arXiv 2410.07112v2, CC BY 4.0), Table 4 'The mean win rate and scores for the individual datasets', row 'Claude 3.0 Haiku', column 'MathVista (EM)' = 0.352. Authors' own HELM runs: 'We evaluate 22 recent models ... implementing consistent adaptation and evaluation methods across all the models'; zero-shot prompting only; exact match. Identity: Table A2 lists 'Claude 3 Haiku (20240307)'.",
    "source_id": "paper-2410.07112v2-vhelm"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.1328,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'scribe_v2' (access proprietary): mean_wer 13.28% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 28.71, CoRal read-aloud 16.95, FT Speech 10.22, Common Voice 17 da 5.49, FLEURS da 5.01); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "gpt-transcribe",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.167,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-10-04",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'gpt-transcribe' (access proprietary): mean_wer 16.7% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 31.57, CoRal read-aloud 20.14, FT Speech 13.89, Common Voice 17 da 10.06, FLEURS da 7.84); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2081,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'mistralai/Voxtral-Small-24B-2507' (access open): mean_wer 20.81% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 39.72, CoRal read-aloud 24.83, FT Speech 16.46, Common Voice 17 da 14.5, FLEURS da 8.52); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2224,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-10-04",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'gpt-4o-mini-transcribe' (access proprietary): mean_wer 22.24% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 41.56, CoRal read-aloud 27.23, FT Speech 17.0, Common Voice 17 da 17.29, FLEURS da 8.13); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "whisper-large-v3",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2389,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'openai/whisper-large-v3' (access open): mean_wer 23.89% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 48.79, CoRal read-aloud 26.87, FT Speech 16.47, Common Voice 17 da 14.51, FLEURS da 12.79); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "canary-1b-v2",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2467,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'nvidia/canary-1b-v2' (access open): mean_wer 24.67% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 46.37, CoRal read-aloud 29.44, FT Speech 20.97, Common Voice 17 da 15.05, FLEURS da 11.52); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2549,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'mistralai/Voxtral-Mini-3B-2507' (access open): mean_wer 25.49% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 46.02, CoRal read-aloud 30.05, FT Speech 20.13, Common Voice 17 da 18.45, FLEURS da 12.82); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "gpt-4o-transcribe",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2614,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-10-04",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'gpt-4o-transcribe' (access proprietary): mean_wer 26.14% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 58.13, CoRal read-aloud 25.11, FT Speech 25.55, Common Voice 17 da 15.6, FLEURS da 6.33); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.2843,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-23",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'openai/whisper-large-v3-turbo' (access open): mean_wer 28.43% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 63.26, CoRal read-aloud 31.33, FT Speech 18.39, Common Voice 17 da 16.24, FLEURS da 12.93); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.295,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'Qwen/Qwen3-ASR-1.7B' (access open): mean_wer 29.5% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 42.14, CoRal read-aloud 37.24, FT Speech 24.13, Common Voice 17 da 23.34, FLEURS da 20.67); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.3069,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-22",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'nvidia/parakeet-tdt-0.6b-v3' (access open): mean_wer 30.69% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 50.38, CoRal read-aloud 40.26, FT Speech 26.15, Common Voice 17 da 18.04, FLEURS da 18.62); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.3218,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'facebook/seamless-m4t-v2-large' (access open): mean_wer 32.18% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 59.31, CoRal read-aloud 28.31, FT Speech 46.84, Common Voice 17 da 15.5, FLEURS da 10.93); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "mms-1b-all",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.3909,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'facebook/mms-1b-all' (access open): mean_wer 39.09% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 61.51, CoRal read-aloud 43.4, FT Speech 44.7, Common Voice 17 da 21.98, FLEURS da 23.86); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "whisper-small",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.469,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'openai/whisper-small' (access open): mean_wer 46.9% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 80.76, CoRal read-aloud 49.49, FT Speech 36.35, Common Voice 17 da 34.39, FLEURS da 33.53); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.4738,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-09-26",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'nvidia/nemotron-3.5-asr-streaming-0.6b' (access open): mean_wer 47.38% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 73.26, CoRal read-aloud 56.67, FT Speech 35.16, Common Voice 17 da 40.87, FLEURS da 30.93); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "whisper-base",
    "bench": "danish-asr-5set-mean-wer",
    "score": 0.9667,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'openai/whisper-base' (access open): mean_wer 96.67% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 210.4, CoRal read-aloud 84.54, FT Speech 59.42, Common Voice 17 da 66.77, FLEURS da 62.23); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "whisper-tiny",
    "bench": "danish-asr-5set-mean-wer",
    "score": 1.3269,
    "source": "https://huggingface.co/datasets/RyeAI/danish-asr-leaderboard/blob/344e5a7047d9b5f3b1c8248050ae5e1dba186f43/data/results.parquet",
    "by": "independent",
    "date": "2026-06-24",
    "note": "Danish ASR Leaderboard results dataset (RyeAI/danish-asr-leaderboard @344e5a7, MIT), data/results.parquet row 'openai/whisper-tiny' (access open): mean_wer 132.69% = macro-average of corpus WER on the five core Danish test sets (CoRal conversation 249.27, CoRal read-aloud 110.44, FT Speech 106.75, Common Voice 17 da 99.59, FLEURS da 97.41); same value in the Space's committed leaderboard.json @feb3f84. Harness: github.com/Rye-A1/danish-asr-leaderboard (MIT) — identical Danish normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) for every system; per-sample hypotheses published under outputs/ in the same dataset. Identity: exact HF repo id / API model name as printed.",
    "source_id": "speech-stt-danish-asr-ryeai-mit"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.0837,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/CohereLabs__cohere-transcribe-03-2026.json",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/CohereLabs__cohere-transcribe-03-2026.json (model_id 'CohereLabs/cohere-transcribe-03-2026', backend vllm, normalize True): WER FLEURS nl 5.8% (364 utt), VoxPopuli nl 9.82% (1,137), MLS nl 9.48% (3,075), 0 failures; Average WER 8.37% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: HF repo CohereLabs/cohere-transcribe-03-2026, vLLM.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.0685,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/ElevenLabs-scribe_v2.json",
    "by": "independent",
    "date": "2026-05-12",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/ElevenLabs-scribe_v2.json (model_id 'scribe_v2', backend api, normalize True): WER FLEURS nl 3.8% (364 utt), VoxPopuli nl 10.93% (1,137), MLS nl 5.83% (3,075), 0 failures; Average WER 6.85% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: ElevenLabs API model scribe_v2.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.1089,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/Qwen__Qwen3-ASR-1.7B.json",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/Qwen__Qwen3-ASR-1.7B.json (model_id 'Qwen/Qwen3-ASR-1.7B', backend vllm, normalize True): WER FLEURS nl 7.06% (364 utt), VoxPopuli nl 13.97% (1,137), MLS nl 11.65% (3,075), 0 failures; Average WER 10.89% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: HF repo Qwen/Qwen3-ASR-1.7B, vLLM.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.1022,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/mistralai__Voxtral-Mini-4B-Realtime-2602.json",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/mistralai__Voxtral-Mini-4B-Realtime-2602.json (model_id 'mistralai/Voxtral-Mini-4B-Realtime-2602', backend vllm, normalize True): WER FLEURS nl 8.42% (364 utt), VoxPopuli nl 13.79% (1,137), MLS nl 8.45% (3,075), 0 failures; Average WER 10.22% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: HF repo mistralai/Voxtral-Mini-4B-Realtime-2602, vLLM.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.101,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/nvidia__parakeet-tdt-0.6b-v3.json",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/nvidia__parakeet-tdt-0.6b-v3.json (model_id 'nvidia/parakeet-tdt-0.6b-v3', backend nemo, normalize True): WER FLEURS nl 7.68% (364 utt), VoxPopuli nl 11.52% (1,137), MLS nl 11.09% (3,075), 0 failures; Average WER 10.10% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: HF repo nvidia/parakeet-tdt-0.6b-v3, NeMo.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "whisper-large-v3",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.1301,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/openai__whisper-large-v3.json",
    "by": "independent",
    "date": "2026-05-11",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/openai__whisper-large-v3.json (model_id 'openai/whisper-large-v3', backend vllm, normalize True): WER FLEURS nl 5.73% (364 utt), VoxPopuli nl 23.53% (1,137), MLS nl 9.78% (3,075), 0 failures; Average WER 13.01% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: HF repo openai/whisper-large-v3, vLLM.",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "voxtral-mini-transcribe-2",
    "bench": "dutch-asr-3set-avg-wer",
    "score": 0.0824,
    "source": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/results/voxtral-mini-2602.json",
    "by": "independent",
    "date": "2026-05-12",
    "note": "Dutch ASR Leaderboard Space (tvosch/Dutch-ASR-Leaderboard @939431c, apache-2.0), committed results/voxtral-mini-2602.json (model_id 'voxtral-mini-2602', backend api, normalize True): WER FLEURS nl 4.38% (364 utt), VoxPopuli nl 12.26% (1,137), MLS nl 8.09% (3,075), 0 failures; Average WER 8.24% = plain mean of the three, as app.py build_dataframe computes ('Average WER' over DATASET_KEYS fleurs_nl/voxpopuli_nl/mls_nl). Per-sample hypotheses are in the same file. Identity: Mistral API model voxtral-mini-2602 (Voxtral Mini Transcribe V2).",
    "source_id": "speech-stt-dutch-asr-tvosch-apache"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.648,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A 'Capability evaluation summary', row 'SWE-Bench Pro', column 'Haiku 5.5' = 64.8 (also §8.2 p.112: 'Claude Haiku 5.5 achieved 64.8%'); standard configuration per caption: adaptive thinking at max effort, default sampling settings, averaged over five trials, context <=1M; public/commercial split not stated (problems 'drawn from actively-maintained repositories with large, multi-file diffs'); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "swe-bench-multilingual",
    "score": 0.837,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'SWE-bench Multilingual', column 'Haiku 5.5' = 83.7 (§8.2 p.112: 300 problems across 9 languages, 'Haiku 5.5 achieved 83.7%'); standard configuration per caption: adaptive thinking at max effort, default sampling settings, averaged over five trials, context <=1M; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "hle",
    "score": 0.459,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Haiku 5.5' = 45.9. §8.8.1: full 2,500-question set, thinking auto at max effort, 980k-token task budget, no compaction, Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "hle-with-tools",
    "score": 0.574,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Haiku 5.5' = 57.4. §8.8.1: web search, web fetch, programmatic tool calling and code execution, 980k-token task budget, no compaction, HLE blocklist, Claude Opus 4.6 grader; max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "terminal-bench-4",
    "score": 0.392,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, §8.4 'Terminal-Bench 4.0' prose: 'Claude Haiku 5.5 scored 39.2% on Terminal-Bench 4.0 with safeguards enabled'; 66 tasks, 10 trials per task (660 trials), SE ±1.9 pts; Claude Code in --bare mode at max thinking effort; no fallback model (1.8% of trials stopped by safeguards, counted as failures); no internet egress (resources pre-cached); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "chartography-with-tools",
    "score": 0.862,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=124",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, §8.9.1 'Chartography' prose: 'Claude Haiku 5.5 scored 46.4% without tools and 86.2% with tools'; with-tools figure recorded (container with the image file, standard libraries and an image cropping tool); adaptive thinking at max effort; internal grading implementation with Gemini 3.5 Flash grader (as on the Surge AI leaderboard); averaged over five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "global-mmlu",
    "score": 0.878,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=134",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, §8.12.1 'GMMLU results' prose: 'Claude Haiku 5.5 achieved an average accuracy of 87.8% across 42 languages' (Global MMLU, full); adaptive thinking at max effort, safety classifiers on, blocked/unanswered questions (0.1%) left out of the average, single trial, no tools or custom system prompt (Figure 8.12.1.A caption); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "swe-bench-multilingual",
    "score": 0.674,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'SWE-bench Multilingual', column 'Haiku 4.5' = 67.4 (Anthropic's own earlier model, same-lab comparison column); averaged over five trials; Claude Haiku 4.5 has no effort setting (run with extended thinking); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "hle",
    "score": 0.102,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'No tools', column 'Haiku 4.5' = 10.2 (same-lab column). §8.8.1: Claude Haiku 4.5 has no effort setting, so it was run at several extended-thinking budgets; Claude Opus 4.6 grader; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "hle-with-tools",
    "score": 0.187,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=111",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, Table 8.1.A, row 'Humanity's Last Exam' / 'With tools', column 'Haiku 4.5' = 18.7 (same-lab column). §8.8.1: result at a 60K-token extended-thinking budget (close to its 64K output limit) with a 160K-token task budget to fit its 200K context; web search, web fetch, programmatic tool calling, code execution; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "terminal-bench-4",
    "score": 0,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=115",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, §8.4 'Terminal-Bench 4.0' prose: 'Claude Haiku 4.5 scored 0.0%, passing none of its trials'; 10 trials per task (660 trials); Claude Code in --bare mode with a fixed thinking budget of 63,999 tokens ('as it predates effort levels'). A genuine floor result on a frontier agentic test (not a parse failure: the card reports trials run and failed); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "chartography-with-tools",
    "score": 0.088,
    "source": "https://www-cdn.anthropic.com/e1080d6bf5ae2018ea3c2f414064be03232f5be5/Claude%20Haiku%205.5%20System%20Card.pdf#page=124",
    "by": "lab",
    "date": "2026-10-07",
    "note": "Anthropic Claude Haiku 5.5 System Card, §8.9.1 'Chartography' prose: 'Claude Haiku 4.5 scored 6.4% without tools and 8.8% with tools'; with-tools figure recorded; Haiku 4.5 run at fixed extended-thinking budgets up to 32k (Figure 8.9.1.A caption); same tools and Gemini 3.5 Flash grader as the other models; five runs; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.778,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'SWE-bench Pro', column 'Mythos Preview' = 77.8 (same value as the Mythos Preview System Card Table 6.3.A); standard configuration (adaptive thinking, max effort, five trials); split not stated; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "swe-bench-verified",
    "score": 0.939,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'SWE-bench Verified', column 'Mythos Preview' = 93.9 (same in the Mythos Preview System Card Table 6.3.A); adaptive thinking at max effort, averaged over five trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "hle",
    "score": 0.568,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'Humanity's Last Exam' / 'No tools', column 'Mythos Preview' = 56.8 (Mythos Preview System Card §6.10.1: 'scored 56.8% without tools', full 2,500 questions, Claude Opus 4.6 grader, max effort); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "hle-with-tools",
    "score": 0.647,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'Humanity's Last Exam' / 'With tools', column 'Mythos Preview' = 64.7 (Mythos Preview System Card §6.10.1: web search, web fetch, programmatic tool calling, code execution, context compaction every 50k tokens up to 3M tokens, HLE blocklist, Claude Opus 4.6 grader); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "charxiv-rq",
    "score": 0.862,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'CharXiv Reasoning' / 'No tools', column 'Mythos Preview' = 86.2 (newest grader, same table as the existing claude-opus-4-8 80.5 row); the Mythos Preview System Card (Table 6.3.A, older grader) prints 86.1, not used. 1,000 validation questions, adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "osworld-verified",
    "score": 0.854,
    "source": "https://www-cdn.anthropic.com/57a52ea7d8f0e54e8a542e908266086df425cdf5/Claude%20Fable%205%20&%20Claude%20Mythos%205%20System%20Card.pdf#page=251",
    "by": "lab",
    "date": "2026-06-09",
    "note": "Anthropic Claude Fable 5 & Mythos 5 System Card, Table 8.1.A 'Capability evaluation summary' (p.251), row 'OSWorld-Verified', column 'Mythos Preview' = 85.4; §8.16.3 (p.280): Anthropic re-evaluated Mythos Preview with the zoom-tool batched-action fix and a 128K per-turn token limit ('we have been underreporting OSWorld performance on it'); 361 tasks, 1080p, 100 steps, auto-thinking at max effort, pass@1 averaged over five runs (same protocol as the opus-4-8 83.4 row); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "swe-bench-multilingual",
    "score": 0.873,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, Table 6.3.A 'Capability Evaluation Summary', row 'SWE-bench Multilingual', column 'Claude Mythos Preview' = 87.3%; standard configuration: adaptive thinking at max effort, default sampling, averaged over 5 trials; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "usamo-2026",
    "score": 0.976,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=191",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, §6.8 'USAMO 2026': 'Claude Mythos Preview achieved a 97.6% score, averaging over 10 trials per problem using max effort and no tools' (also Table 6.3.A). MathArena methodology: proofs rewritten by Gemini 3.1 Pro, judged by a 3-model panel (Gemini 3.1 Pro, Claude Opus 4.6, Claude Mythos Preview), minimum score; the card notes two of three judges were Anthropic models; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "browsecomp-context-managed",
    "score": 0.869,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=193",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, §6.10.2 'BrowseComp': 'Claude Mythos Preview scored 86.9% with adaptive thinking at max effort and a 3M token limit. We used context compaction (triggered at 200k tokens)'; web search, web fetch, programmatic tool calling, code execution; single agent. The Fable 5 card's later 87.9 (budget not stated) is not used; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "terminal-bench-2",
    "score": 0.82,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=190",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, §6.5 'Terminal-Bench 2.0': 'Claude Mythos Preview achieved 82% mean reward, averaged over 5 attempts for each one of the 89 unique tasks (for a total of 445 trials)'; Harbor scaffold, Terminus-2 harness, default parser, 1x resources and timeouts, max effort (adaptive), 1M total tokens per task, 32K max output per request. The 92.1% figure with 2.1 fixes and 4-hour timeouts is not used; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "gpqa-diamond",
    "score": 0.9455,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=191",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, §6.6 'GPQA Diamond': 'Claude Mythos Preview achieved 94.55% on GPQA Diamond, averaged over 5 trials' (198 questions; Table 6.3.A prints 94.5%); adaptive thinking at max effort; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-mythos-preview",
    "bench": "mmmlu",
    "score": 0.9267,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=191",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, §6.7 'MMMLU': 'achieves 92.67% averaged over 5 trials on all non-English language pairings' (14 languages; Table 6.3.A prints 92.7%); adaptive thinking, max effort, default sampling; vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "terminal-bench-2",
    "score": 0.654,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, Table 6.3.A, row 'Terminal-Bench 2.0*', column 'Claude Opus 4.6' = 65.4% (Anthropic's own model; caption: 'All other scores used the Terminus-2 harness'; earlier-model details per the Claude Opus 4.6 System Card); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "gpqa-diamond",
    "score": 0.913,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=188",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, Table 6.3.A, row 'GPQA Diamond', column 'Claude Opus 4.6' = 91.3% (Anthropic's own model; evaluation details per the Claude Opus 4.6 System Card); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "mmmlu",
    "score": 0.911,
    "source": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf#page=189",
    "by": "lab",
    "date": "2026-04-07",
    "note": "Anthropic Claude Mythos Preview System Card, Table 6.3.A, row 'MMMLU', column 'Claude Opus 4.6' = 91.1% (Anthropic's own model; evaluation details per the Claude Opus 4.6 System Card); vendor-reported (rule b)",
    "source_id": "first-party-anthropic-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "deepswe-v1-1",
    "score": 0.578,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'DeepSWE 1.1', column 'Qwen3.8-Omni-Flash' = 57.8; reasoning setting not printed (API default reasoning_effort xhigh); footnote 1: 'evaluated with the Claude Code and mini-SWE-agent harnesses, temp=1.0, top_p=0.95, 256K context window. We report the highest score across the two harnesses'; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "deepswe-v1-1",
    "score": 0.422,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'DeepSWE 1.1', column 'Qwen3.8-27B' = 42.2; reasoning setting not printed (API default reasoning_effort xhigh); footnote 1: 'evaluated with the Claude Code and mini-SWE-agent harnesses, temp=1.0, top_p=0.95, 256K context window. We report the highest score across the two harnesses'. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "deepswe-v1-1",
    "score": 0.165,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'DeepSWE 1.1', column 'Qwen3.7-Plus' = 16.5; reasoning setting not printed (API default reasoning_effort xhigh); footnote 1: 'evaluated with the Claude Code and mini-SWE-agent harnesses, temp=1.0, top_p=0.95, 256K context window. We report the highest score across the two harnesses'. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.633,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Pro', column 'Qwen3.8-Omni-Flash' = 63.3; reasoning setting not printed (API default reasoning_effort xhigh); footnote 2: 'all models are evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context window. Problematic tasks were corrected and all baseline models were re-evaluated on the refined benchmark' (Qwen-refined public set); vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.617,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Pro', column 'Qwen3.8-27B' = 61.7; reasoning setting not printed (API default reasoning_effort xhigh); footnote 2: 'all models are evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context window. Problematic tasks were corrected and all baseline models were re-evaluated on the refined benchmark' (Qwen-refined public set). Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.558,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Pro', column 'Qwen3.7-Plus' = 55.8; reasoning setting not printed (API default reasoning_effort xhigh); footnote 2: 'all models are evaluated with the Claude Code harness, temp=1.0, top_p=0.95, 256K context window. Problematic tasks were corrected and all baseline models were re-evaluated on the refined benchmark' (Qwen-refined public set). Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "swe-bench-multilingual",
    "score": 0.805,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Multilingual', column 'Qwen3.8-Omni-Flash' = 80.5; reasoning setting not printed (API default reasoning_effort xhigh); harness not stated for this row; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "swe-bench-multilingual",
    "score": 0.738,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Multilingual', column 'Qwen3.8-27B' = 73.8; reasoning setting not printed (API default reasoning_effort xhigh); harness not stated for this row. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "swe-bench-multilingual",
    "score": 0.758,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Multilingual', column 'Qwen3.7-Plus' = 75.8; reasoning setting not printed (API default reasoning_effort xhigh); harness not stated for this row. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "ifbench-unspecified",
    "score": 0.815,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'IFBench', column 'Qwen3.8-Omni-Flash' = 81.5; reasoning setting not printed (API default reasoning_effort xhigh); aggregation not stated; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "ifbench-unspecified",
    "score": 0.795,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'IFBench', column 'Qwen3.8-27B' = 79.5; reasoning setting not printed (API default reasoning_effort xhigh); aggregation not stated. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "ifbench-unspecified",
    "score": 0.791,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'IFBench', column 'Qwen3.7-Plus' = 79.1; reasoning setting not printed (API default reasoning_effort xhigh); aggregation not stated. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "gpqa-diamond",
    "score": 0.91,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'GPQA Diamond', column 'Qwen3.8-Omni-Flash' = 91.0; reasoning setting not printed (API default reasoning_effort xhigh); vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "gpqa-diamond",
    "score": 0.892,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'GPQA Diamond', column 'Qwen3.8-27B' = 89.2; reasoning setting not printed (API default reasoning_effort xhigh). Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "gpqa-diamond",
    "score": 0.903,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'GPQA Diamond', column 'Qwen3.7-Plus' = 90.3; reasoning setting not printed (API default reasoning_effort xhigh). Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "hle",
    "score": 0.365,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'HLE', column 'Qwen3.8-Omni-Flash' = 36.5; reasoning setting not printed (API default reasoning_effort xhigh); no tools stated or implied by a tool footnote; text vs full set not stated; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "hle",
    "score": 0.308,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'HLE', column 'Qwen3.8-27B' = 30.8; reasoning setting not printed (API default reasoning_effort xhigh); no tools stated or implied by a tool footnote; text vs full set not stated. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "hle",
    "score": 0.347,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'HLE', column 'Qwen3.7-Plus' = 34.7; reasoning setting not printed (API default reasoning_effort xhigh); no tools stated or implied by a tool footnote; text vs full set not stated. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.926,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'LiveCodeBench v6', column 'Qwen3.8-Omni-Flash' = 92.6; reasoning setting not printed (API default reasoning_effort xhigh); date window not printed; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.903,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'LiveCodeBench v6', column 'Qwen3.8-27B' = 90.3; reasoning setting not printed (API default reasoning_effort xhigh); date window not printed. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.896,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'LiveCodeBench v6', column 'Qwen3.7-Plus' = 89.6; reasoning setting not printed (API default reasoning_effort xhigh); date window not printed. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b). Note: the Qwen3.8-27B card (card-qwen--qwen3.8-27b) prints the same figures for several of these cells; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-omni-flash",
    "bench": "charxiv-rq",
    "score": 0.835,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), vision table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | Claude-Opus-4.6 (Max)), row 'CharXiv (RQ)', column 'Qwen3.8-Omni-Flash': 'Without CI 83.5 / With CI 91.4' -> without code interpreter 83.5 recorded, matching the without-CI convention of the other first-party-qwen-scores CharXiv rows; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "charxiv-rq",
    "score": 0.837,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?path=qwen3.8-omni-flash), vision table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | Claude-Opus-4.6 (Max)), row 'CharXiv (RQ)', column 'Qwen3.8-27B': 'Without CI 83.7 / With CI 90.2' -> 83.7 (without CI) recorded. Same-lab column (Qwen's own model) in Qwen's own table; only Qwen columns used, DeepSeek and Claude cells excluded (rule b).; vendor-reported (rule b)",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "kat-coder-v2-5-dev",
    "bench": "swe-bench-verified",
    "score": 0.694,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "lab",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'KAT-Coder-V2.5-Dev' = 69.4. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-verified",
    "score": 0.686,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Qwen3.5-27B' = 68.6. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.644,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Qwen3.6-35BA3B' = 64.4. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: on SWE-bench Verified/Multilingual/Pro 'our test results this time have an approximate 10 pp gap compared with the official results' (harness version).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-verified",
    "score": 0.606,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Gemma4-31B' = 60.6. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.586,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Qwen3.5-35BA3B' = 58.6. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'frequent hallucinations ... attempts to invoke the unavailable MultiEdit tool ... negatively impacts the final metric'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "swe-bench-verified",
    "score": 0.358,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Gemma4-26BA4B' = 35.8. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'context overflow (exceeding the 256k context limit) and hallucinated calls to the unsupported MultiEdit tool'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.318,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Qwen3-Coder-30B' = 31.8. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "kat-coder-v2-5-dev",
    "bench": "swe-bench-multilingual",
    "score": 0.63,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "lab",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'KAT-Coder-V2.5-Dev' = 63.0. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-multilingual",
    "score": 0.5767,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Qwen3.5-27B' = 57.67. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.57,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Qwen3.6-35BA3B' = 57.0. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: on SWE-bench Verified/Multilingual/Pro 'our test results this time have an approximate 10 pp gap compared with the official results' (harness version).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-multilingual",
    "score": 0.4933,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Gemma4-31B' = 49.33. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.4767,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Qwen3.5-35BA3B' = 47.67. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'frequent hallucinations ... attempts to invoke the unavailable MultiEdit tool ... negatively impacts the final metric'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "swe-bench-multilingual",
    "score": 0.2733,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Gemma4-26BA4B' = 27.33. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'context overflow (exceeding the 256k context limit) and hallucinated calls to the unsupported MultiEdit tool'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.2067,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Qwen3-Coder-30B' = 20.67. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "kat-coder-v2-5-dev",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.4596,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "lab",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'KAT-Coder-V2.5-Dev' = 45.96. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.4213,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Qwen3.5-27B' = 42.13. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.4063,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Qwen3.6-35BA3B' = 40.63. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: on SWE-bench Verified/Multilingual/Pro 'our test results this time have an approximate 10 pp gap compared with the official results' (harness version).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-31b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.3297,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Gemma4-31B' = 32.97. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.3803,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Qwen3.5-35BA3B' = 38.03. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'frequent hallucinations ... attempts to invoke the unavailable MultiEdit tool ... negatively impacts the final metric'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.0958,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Gemma4-26BA4B' = 9.58. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Card anomaly note: 'context overflow (exceeding the 256k context limit) and hallucinated calls to the unsupported MultiEdit tool'.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.1984,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Qwen3-Coder-30B' = 19.84. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "kat-coder-v2-5-dev",
    "bench": "terminal-bench-2.1",
    "score": 0.326,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "lab",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'KAT-Coder-V2.5-Dev' = 32.6. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "terminal-bench-2.1",
    "score": 0.4157,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Qwen3.5-27B' = 41.57. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "terminal-bench-2.1",
    "score": 0.3483,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Qwen3.6-35BA3B' = 34.83. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-31b",
    "bench": "terminal-bench-2.1",
    "score": 0.3034,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Gemma4-31B' = 30.34. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "terminal-bench-2.1",
    "score": 0.2644,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Qwen3.5-35BA3B' = 26.44. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "terminal-bench-2.1",
    "score": 0.2727,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Gemma4-26BA4B' = 27.27. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "terminal-bench-2.1",
    "score": 0.1011,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Qwen3-Coder-30B' = 10.11. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code'; the bold two-harness average is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "kat-coder-v2-5-dev",
    "bench": "scicode-unspecified",
    "score": 0.442,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "lab",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'KAT-Coder-V2.5-Dev' = 44.2. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "scicode-unspecified",
    "score": 0.2558,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Qwen3.5-27B' = 25.58. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "scicode-unspecified",
    "score": 0.3753,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Qwen3.6-35BA3B' = 37.53. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-31b",
    "bench": "scicode-unspecified",
    "score": 0.3319,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Gemma4-31B' = 33.19. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "scicode-unspecified",
    "score": 0.2773,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Qwen3.5-35BA3B' = 27.73. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "scicode-unspecified",
    "score": 0.3084,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Gemma4-26BA4B' = 30.84. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "qwen3-coder-30b-a3b",
    "bench": "scicode-unspecified",
    "score": 0.1827,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Qwen3-Coder-30B' = 18.27. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once.",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "terminal-bench-2.1",
    "score": 0.839,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Terminal Bench 2.1', column 'DeepSeek-V4-Flash-Vision-Exp' = 83.9. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.827,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Terminal Bench 2.1', column 'DeepSeek-V4-Flash-0731' = 82.7. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "nl2repo",
    "score": 0.577,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'NL2Repo', column 'DeepSeek-V4-Flash-Vision-Exp' = 57.7. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "nl2repo",
    "score": 0.542,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'NL2Repo', column 'DeepSeek-V4-Flash-0731' = 54.2. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "cybergym",
    "score": 0.753,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Cybergym', column 'DeepSeek-V4-Flash-Vision-Exp' = 75.3. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "cybergym",
    "score": 0.767,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Cybergym', column 'DeepSeek-V4-Flash-0731' = 76.7. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "deepswe-v1-1",
    "score": 0.593,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'DeepSWE', column 'DeepSeek-V4-Flash-Vision-Exp' = 59.3. DeepSWE (version not printed in this card; DeepSeek's V4.1-Flash card labels it DeepSWE v1.1 and prints the same 0731 value 54.4); DeepSeek Harness minimal mode, max effort. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "deepswe-v1-1",
    "score": 0.544,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'DeepSWE', column 'DeepSeek-V4-Flash-0731' = 54.4. DeepSWE (version not printed in this card; DeepSeek's V4.1-Flash card labels it DeepSWE v1.1 and prints the same 0731 value 54.4); DeepSeek Harness minimal mode, max effort. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "toolathlon-verified",
    "score": 0.759,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Toolathlon-Verified', column 'DeepSeek-V4-Flash-Vision-Exp' = 75.9. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "toolathlon-verified",
    "score": 0.703,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Toolathlon-Verified', column 'DeepSeek-V4-Flash-0731' = 70.3. DeepSeek Harness minimal mode, max reasoning effort, temperature 1.0, top_p 0.95. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "automationbench-public",
    "score": 0.257,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'AutomationBench (Public)', column 'DeepSeek-V4-Flash-Vision-Exp' = 25.7. label as in DeepSeek's V4-Pro-0813 card; DeepSeek Harness minimal mode, max effort. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "automationbench-public",
    "score": 0.251,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'AutomationBench (Public)', column 'DeepSeek-V4-Flash-0731' = 25.1. label as in DeepSeek's V4-Pro-0813 card; DeepSeek Harness minimal mode, max effort. Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "agents-last-exam-unspecified",
    "score": 0.273,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Agents' Last Exam', column 'DeepSeek-V4-Flash-Vision-Exp' = 27.3. multimodal-agent block; harness/metric not stated for this block (note 1 covers text agent benchmarks only). Card note 2: '† For ApexBench and Agents' Last Exam, DeepSeek-V4-Flash-0731 ignores the multimodal elements in the input' (the 0731 cell carries †). Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "agents-last-exam-unspecified",
    "score": 0.252,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/README.md",
    "by": "lab",
    "date": "2026-08-31",
    "note": "DeepSeek-V4-Flash-Vision-Exp model card (MIT, pinned 6821d6a), Introduction table, row 'Agents' Last Exam', column 'DeepSeek-V4-Flash-0731' = 25.2. multimodal-agent block; harness/metric not stated for this block (note 1 covers text agent benchmarks only). Card note 2: '† For ApexBench and Agents' Last Exam, DeepSeek-V4-Flash-0731 ignores the multimodal elements in the input' (the 0731 cell carries †). Card note 1: 'DeepSeek models are evaluated with the minimal mode of DeepSeek Harness as the agent framework, using the max reasoning effort level with temperature = 1.0, top_p = 0.95.' Same-lab columns; the Opus-4.8 column is not used (no statement that DeepSeek ran it). Checkpoint: 'DeepSeek-V4-Flash-0731' = catalogued deepseek-v4-flash.",
    "source_id": "card3-deepseek-ai-deepseek-v4-flash-vision-exp"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "lmarena-text",
    "score": 1451.8,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'mimo-v2.6-flash'; 6,053 votes; published 95% interval 1443.9–1459.6. Same pinned October revision as the other lmarena-text rows; read with pyarrow 2026-10-09.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "lmarena-vision",
    "score": 1247.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/vision_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest; exact entry 'mimo-v2.6-flash'; 2,275 votes; published 95% interval 1234.2–1260.7. Same pinned October revision as the other lmarena-vision rows; read with pyarrow 2026-10-09.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "lmarena-webdev",
    "score": 1635.7,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/ff77a170d5ec578fe4eb8c91e0084a6b1a20beac/webdev/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "webdev, category overall, split latest; exact entry 'mimo-v2.6-flash'; 3,851 votes; published 95% interval 1625.1–1646.3. Entry absent from the 46919c4 webdev split; taken from the next webdev update ff77a17 ('Update webdev for 2026-10-07', publish date 2026-10-06), which already supplies two catalogue rows (needs add_match for this file).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "deepswe-v1-1",
    "score": 0.719,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'DeepSWE v1.1', column 'MiMo-V2.6 Pro' = 71.9. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "deepswe-v1-1",
    "score": 0.679,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'DeepSWE v1.1', column 'MiMo-V2.6 Flash' = 67.9. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "deepswe-v1-1",
    "score": 0.19,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'DeepSWE v1.1', column 'MiMo-V2.5 Pro' = 19.0. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "toolathlon-verified",
    "score": 0.769,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Toolathlon-Verified', column 'MiMo-V2.6 Pro' = 76.9. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. . Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "toolathlon-verified",
    "score": 0.736,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Toolathlon-Verified', column 'MiMo-V2.6 Flash' = 73.6. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. . Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "toolathlon-verified",
    "score": 0.491,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Toolathlon-Verified', column 'MiMo-V2.5 Pro' = 49.1. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. . Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "agents-last-exam-unspecified",
    "score": 0.316,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Agents’ Last Exam', column 'MiMo-V2.6 Pro' = 31.6. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric and harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "agents-last-exam-unspecified",
    "score": 0.276,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Agents’ Last Exam', column 'MiMo-V2.6 Flash' = 27.6. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric and harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "agents-last-exam-unspecified",
    "score": 0.132,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Agents’ Last Exam', column 'MiMo-V2.5 Pro' = 13.2. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric and harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "terminal-bench-4",
    "score": 0.349,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 4.0', column 'MiMo-V2.6 Pro' = 34.9. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "terminal-bench-4",
    "score": 0.288,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 4.0', column 'MiMo-V2.6 Flash' = 28.8. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "terminal-bench-4",
    "score": 0.015,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 4.0', column 'MiMo-V2.5 Pro' = 1.5. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "terminal-bench-2.1",
    "score": 0.899,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 2.1', column 'MiMo-V2.6 Pro' = 89.9. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "terminal-bench-2.1",
    "score": 0.876,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 2.1', column 'MiMo-V2.6 Flash' = 87.6. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "terminal-bench-2.1",
    "score": 0.652,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'Terminal Bench 2.1', column 'MiMo-V2.5 Pro' = 65.2. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. harness not stated. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "osworld-verified",
    "score": 0.82,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'OSWorld-Verified', column 'MiMo-V2.6 Pro' = 82.0. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. V2.5 Pro cell is '-'. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "osworld-verified",
    "score": 0.808,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'OSWorld-Verified', column 'MiMo-V2.6 Flash' = 80.8. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. V2.5 Pro cell is '-'. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "automationbench-v1-0-6",
    "score": 0.531,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'AutomationBench v1.0.6', column 'MiMo-V2.6 Pro' = 53.1. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. version v1.0.6 as printed; the PR #13 null-type fix is not mentioned. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "automationbench-v1-0-6",
    "score": 0.523,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'AutomationBench v1.0.6', column 'MiMo-V2.6 Flash' = 52.3. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. version v1.0.6 as printed; the PR #13 null-type fix is not mentioned. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "automationbench-v1-0-6",
    "score": 0.16,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'AutomationBench v1.0.6', column 'MiMo-V2.5 Pro' = 16.0. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. version v1.0.6 as printed; the PR #13 null-type fix is not mentioned. Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "exploitgym",
    "score": 0.178,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'ExploitGym', column 'MiMo-V2.6 Pro' = 17.8. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "exploitgym",
    "score": 0.06,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'ExploitGym', column 'MiMo-V2.6 Flash' = 6.0. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "exploitgym",
    "score": 0.002,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'ExploitGym', column 'MiMo-V2.5 Pro' = 0.2. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "sec-bench-pro",
    "score": 0.663,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'SEC Bench Pro', column 'MiMo-V2.6 Pro' = 66.3. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "sec-bench-pro",
    "score": 0.475,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'SEC Bench Pro', column 'MiMo-V2.6 Flash' = 47.5. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "sec-bench-pro",
    "score": 0.177,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/5711b268169967567844e1e560e8a3966da959b1/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Flash-RL model card (MIT, pinned 5711b26), '## 3. Evaluation Results' table, row 'SEC Bench Pro', column 'MiMo-V2.5 Pro' = 17.7. Same table also in the MiMo-V2.6 technical report Table 3 ('Comparison of MiMo-V2.6 with previous-generation and frontier models on agentic benchmarks'); same-lab columns (Xiaomi's own models). Highest setting: report §5.2 'All baseline models with configurable reasoning effort are evaluated at the highest supported setting (max)'; own models at default thinking. metric not printed (Xiaomi reports avg@n elsewhere, i.e. mean pass rate). Checkpoint: 'MiMo-V2.6 Flash' = MiMo-V2.6-Flash-RL (this card), 'MiMo-V2.6 Pro' = MiMo-V2.6-Pro-RL, 'MiMo-V2.5 Pro' = catalogued mimo-v2-5-pro.",
    "source_id": "first-party-xiaomi"
  },
  {
    "model": "mimo-v2-6-distill-qwen-9b",
    "bench": "swe-bench-verified",
    "score": 0.611,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'SWE Verified', column 'MiMo-V2.6-Distill-Qwen-9B (SFT)' = 61.1; metric avg@3. Thinking enabled per card quickstart. Lone in its run unless the held Qwen3.5-9B column is accepted.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "mimo-v2-6-distill-qwen-9b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.446,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'SWE Pro', column 'MiMo-V2.6-Distill-Qwen-9B (SFT)' = 44.6; metric avg@3; split (public/commercial) not stated. Thinking enabled per card quickstart. Lone in its run unless the held Qwen3.5-9B column is accepted.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "mimo-v2-6-distill-qwen-9b",
    "bench": "terminal-bench-2.1",
    "score": 0.371,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'Terminal Bench 2.1', column 'MiMo-V2.6-Distill-Qwen-9B (SFT)' = 37.1; metric avg@1. Thinking enabled per card quickstart. Lone in its run unless the held Qwen3.5-9B column is accepted.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "mimo-v2-6-distill-qwen-9b",
    "bench": "toolathlon-verified",
    "score": 0.352,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'Toolathlon-Verified', column 'MiMo-V2.6-Distill-Qwen-9B (SFT)' = 35.2; metric avg@1. Thinking enabled per card quickstart. Lone in its run unless the held Qwen3.5-9B column is accepted.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "mimo-v2-6-distill-qwen-9b",
    "bench": "automationbench-v1-0-6",
    "score": 0.303,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "lab",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'AutomationBench v1.0.6', column 'MiMo-V2.6-Distill-Qwen-9B (SFT)' = 30.3; metric avg@1; PR #13 fix not mentioned. Thinking enabled per card quickstart. Lone in its run unless the held Qwen3.5-9B column is accepted.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "north-small-translate-1-0",
    "bench": "wmt26-gemba-esa",
    "score": 0.836,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'North Small Translate', column 'GEMBA WMT26' = 83.6. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Plain model row used; the separate 'North Small Translate (Agentic)' row (84.4 / 80.7 / 87.9) is an agentic inference-time workflow, not a different checkpoint, and is not recorded. Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "north-small-translate-1-0",
    "bench": "wmt24pp",
    "score": 0.806,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'North Small Translate', column 'xCOMETxl WMT24++' = 80.6. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Plain model row used; the separate 'North Small Translate (Agentic)' row (84.4 / 80.7 / 87.9) is an agentic inference-time workflow, not a different checkpoint, and is not recorded. Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "north-small-translate-1-0",
    "bench": "wmt25-terminology",
    "score": 0.897,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'North Small Translate', column 'Terminology' = 89.7. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Plain model row used; the separate 'North Small Translate (Agentic)' row (84.4 / 80.7 / 87.9) is an agentic inference-time workflow, not a different checkpoint, and is not recorded. Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-translate-08-2025",
    "bench": "wmt26-gemba-esa",
    "score": 0.7,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A Translate (2025)', column 'GEMBA WMT26' = 70.0. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Row label 'Command A Translate (2025)', 111B (111B): Cohere's only Command A Translate release is command-a-translate-08-2025. Reasoning setting not stated. WMT26 General MT test set, all languages; reference-free GPT-5.6 Sol judge (GEMBA-ESA), mean of adequacy and fluency, segment-weighted per document.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-translate-08-2025",
    "bench": "wmt24pp",
    "score": 0.769,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A Translate (2025)', column 'xCOMETxl WMT24++' = 76.9. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Row label 'Command A Translate (2025)', 111B (111B): Cohere's only Command A Translate release is command-a-translate-08-2025. Reasoning setting not stated. Metric xCOMET-XL (not XXL).",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "command-a-translate-08-2025",
    "bench": "wmt25-terminology",
    "score": 0.569,
    "source": "https://arxiv.org/html/2609.13916v1",
    "by": "lab",
    "date": "2026-09-12",
    "note": "Cohere, 'North Small Translate: Advanced Cost-Effective Translation (Cohere CAT+)', arXiv:2609.13916v1 (12 Sep 2026), CC BY 4.0, Table 1, row 'Command A Translate (2025)', column 'Terminology' = 56.9. §3: 'We evaluate all systems in an identical setup, in a clean zero-shot approach without any inference techniques' (Cohere's own run of its own model). Row label 'Command A Translate (2025)', 111B (111B): Cohere's only Command A Translate release is command-a-translate-08-2025. Reasoning setting not stated. WMT25 Terminology Translation Task, proper-terminology condition, en→de/es/ru/zh-Hant and zh-Hant→en; corpus-level term success rate, micro-averaged over both tracks.",
    "source_id": "paper-2609.13916v1-north-small-translate"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.687,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'MMBench_DEV_EN_V11', cell '0.687' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "mmstar",
    "score": 0.518,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'MMStar', cell '0.518' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "realworldqa",
    "score": 0.622,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'RealWorldQA', cell '0.622' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "chartqa",
    "score": 0.808,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'ChartQA_Test', cell '0.808' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "ocrbench",
    "score": 0.792,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'OCRBench (0-1 scale = raw/1000)', cell '0.792' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "ai2d-test",
    "score": 0.775,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'AI2D_TEST', cell '0.775' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "charxiv-dq",
    "score": 0.6,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'CharXiv_DQ', cell '0.600' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "countbench",
    "score": 0.725,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'CountBench', cell '0.725' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "hallusionbench",
    "score": 0.615,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'HallusionBench', cell '0.615' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "mmlu",
    "score": 0.504,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'MMLU_test (text-only)', cell '0.504' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "mmlu-pro",
    "score": 0.307,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'MMLU-Pro_test (text-only)', cell '0.307' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "multi-if-unspecified",
    "score": 0.373,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'Multi-If (text-only; aggregation not stated)', cell '0.373' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "north-micro-vision-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.749,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "lab",
    "date": "2026-08-10",
    "note": "Column 'North-Micro-Vision-Instruct', row 'IFEval (text-only; aggregation not stated)', cell '0.749' of the Benchmark Results table in Cohere's pinned model card (Apache-2.0). Card: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Non-reasoning instruct model. Same row-to-id mapping as the comparator rows from this card. Values printed as fractions.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-verified",
    "score": 0.558,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Verified', column 'Ornith-1.0-35B' = 55.80. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Checkpoint: ornith-ai/Ornith-1.0-35B (DeepReinforce; the only Ornith 1.0 35B release).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-multilingual",
    "score": 0.5167,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Multilingual', column 'Ornith-1.0-35B' = 51.67. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Checkpoint: ornith-ai/Ornith-1.0-35B (DeepReinforce; the only Ornith 1.0 35B release).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.3447,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'SWE-bench Pro', column 'Ornith-1.0-35B' = 34.47. agent=claude_code@2.1.195, pass@k=1, temperature=1.0, top_p=0.95, 256k ctx; public/commercial split not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Checkpoint: ornith-ai/Ornith-1.0-35B (DeepReinforce; the only Ornith 1.0 35B release).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "terminal-bench-2.1",
    "score": 0.3596,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Terminal-Bench 2.1', column 'Ornith-1.0-35B' = 35.96. Terminus-2 harness value (first of the two per-harness numbers 'Terminus-2 / Claude Code' = 35.96 / 36.00; the bold two-harness average 35.98 is not used), pass@k=1, temperature=0.7, top_p=1.0, 256k ctx. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Checkpoint: ornith-ai/Ornith-1.0-35B (DeepReinforce; the only Ornith 1.0 35B release).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "scicode-unspecified",
    "score": 0.3034,
    "source": "https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/blob/7be56fe773e72b6f5ca93c1ae45d828ddb893922/README.md",
    "by": "independent",
    "date": "2026-07-23",
    "note": "Kwaipilot KAT-Coder-V2.5-Dev model card (Apache-2.0, pinned 7be56fe), 'Benchmark performance' table, row 'Scicode', column 'Ornith-1.0-35B' = 30.34. pass@k=1, temperature=0.6, top_p=1.0, 256k ctx; main-vs-subproblem not stated. Card note 1: 'All metrics presented in the table are reproduced in-house: we download the public model checkpoints, deploy them via vLLM or SGLang, and evaluate under a unified standardized pipeline. No officially reported results of the respective models are directly adopted in this table.' Each model tested once. Checkpoint: ornith-ai/Ornith-1.0-35B (DeepReinforce; the only Ornith 1.0 35B release).",
    "source_id": "card3-kwaipilot-kat-coder-v2-5-dev"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "terminal-bench-2.1",
    "score": 0.678,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'Terminal-Bench 2.1 (Terminus-2)', column 'Ornith-1.5-35B-A3B' = 67.8. Harbor/Terminus-2, parser=json, temperature=1.0, top_p=1.0, 128K context, averaged over 5 runs (card footnote); the Claude Code harness row (68.5 / 62.8) is not used. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "terminal-bench-2.1",
    "score": 0.642,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'Terminal-Bench 2.1 (Terminus-2)', column 'Ornith-1.0-35B-A3B' = 64.2. Harbor/Terminus-2, parser=json, temperature=1.0, top_p=1.0, 128K context, averaged over 5 runs (card footnote); the Claude Code harness row (68.5 / 62.8) is not used. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "swe-bench-verified",
    "score": 0.79,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Verified', column 'Ornith-1.5-35B-A3B' = 79. OpenHands harness, temp=1.0, top_p=0.95, 256k context, anti-hacking safeguards (card footnote). 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-verified",
    "score": 0.756,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Verified', column 'Ornith-1.0-35B-A3B' = 75.6. OpenHands harness, temp=1.0, top_p=0.95, 256k context, anti-hacking safeguards (card footnote). 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.596,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Pro', column 'Ornith-1.5-35B-A3B' = 59.6. OpenHands harness, temp=1.0, top_p=0.95, 256k context; public/commercial split not stated. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.504,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Pro', column 'Ornith-1.0-35B-A3B' = 50.4. OpenHands harness, temp=1.0, top_p=0.95, 256k context; public/commercial split not stated. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "swe-bench-multilingual",
    "score": 0.714,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Multilingual', column 'Ornith-1.5-35B-A3B' = 71.4. OpenHands harness, temp=1.0, top_p=0.95, 256k context. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "swe-bench-multilingual",
    "score": 0.693,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'SWE-bench Multilingual', column 'Ornith-1.0-35B-A3B' = 69.3. OpenHands harness, temp=1.0, top_p=0.95, 256k context. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "nl2repo",
    "score": 0.462,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'NL2Repo', column 'Ornith-1.5-35B-A3B' = 46.2. Card footnote: 'NL2Repo: with temperature=1.0, top_p=1.0, 400K context, 48K output. Access to specified GitHub repositories and pip packages is blocked to prevent reward hacking.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "nl2repo",
    "score": 0.346,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'NL2Repo', column 'Ornith-1.0-35B-A3B' = 34.6. Card footnote: 'NL2Repo: with temperature=1.0, top_p=1.0, 400K context, 48K output. Access to specified GitHub repositories and pip packages is blocked to prevent reward hacking.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "hle",
    "score": 0.256,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'HLE (no tools)', column 'Ornith-1.5-35B-A3B' = 25.6. Card footnote: 'HLE: Evaluated using Claude 4.6 Opus as the judge model.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "hle",
    "score": 0.208,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'HLE (no tools)', column 'Ornith-1.0-35B-A3B' = 20.8. Card footnote: 'HLE: Evaluated using Claude 4.6 Opus as the judge model.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "hle-with-tools",
    "score": 0.334,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'HLE (with tools)', column 'Ornith-1.5-35B-A3B' = 33.4. Card footnote: judge Claude 4.6 Opus; tool set not detailed. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "hle-with-tools",
    "score": 0.301,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'HLE (with tools)', column 'Ornith-1.0-35B-A3B' = 30.1. Card footnote: judge Claude 4.6 Opus; tool set not detailed. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "gpqa-diamond",
    "score": 0.892,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'GPQA Diamond', column 'Ornith-1.5-35B-A3B' = 89.2. Reasoning model, default thinking. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "gpqa-diamond",
    "score": 0.862,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'GPQA Diamond', column 'Ornith-1.0-35B-A3B' = 86.2. Reasoning model, default thinking. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "mcp-atlas",
    "score": 0.702,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'MCP-Atlas', column 'Ornith-1.5-35B-A3B' = 70.2. Card footnote: 'All models were evaluated in thinking mode on the 500-task public subset, with a 10-minute timeout per task. We use Claude 4.8 Opus as the judge model.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "mcp-atlas",
    "score": 0.644,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'MCP-Atlas', column 'Ornith-1.0-35B-A3B' = 64.4. Card footnote: 'All models were evaluated in thinking mode on the 500-task public subset, with a 10-minute timeout per task. We use Claude 4.8 Opus as the judge model.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "toolathlon-verified",
    "score": 0.487,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'Toolathlon-Verified', column 'Ornith-1.5-35B-A3B' = 48.7. Card footnote: 'We use the official evaluation service with the maximum token limit set to 128K.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "toolathlon-verified",
    "score": 0.424,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'Toolathlon-Verified', column 'Ornith-1.0-35B-A3B' = 42.4. Card footnote: 'We use the official evaluation service with the maximum token limit set to 128K.' 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "widesearch",
    "score": 0.678,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'WideSearch', column 'Ornith-1.5-35B-A3B' = 67.8. Metric as printed by the card (not specified). 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "widesearch",
    "score": 0.634,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'WideSearch', column 'Ornith-1.0-35B-A3B' = 63.4. Metric as printed by the card (not specified). 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-5-35b-a3b",
    "bench": "browsecomp",
    "score": 0.676,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'BrowseComp', column 'Ornith-1.5-35B-A3B' = 67.6. Harness/context management not detailed in the card. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "ornith-1-0-35b",
    "bench": "browsecomp",
    "score": 0.635,
    "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/10fbf86fed7ecee4a061f8b499a618f46001cac1/README.md",
    "by": "lab",
    "date": "2026-08-18",
    "note": "DeepReinforce Ornith-1.5-35B-A3B model card (MIT, pinned 10fbf86), benchmark table, row 'BrowseComp', column 'Ornith-1.0-35B-A3B' = 63.5. Harness/context management not detailed in the card. 'All results reported for Ornith-1.5 are averaged over five independent runs.' Same-lab sibling columns (Ornith 1.5 and Ornith 1.0); other makers' columns not used (no statement that the authors ran them, and several match those makers' published figures). Ornith-1.0-35B-A3B = ornith-ai/Ornith-1.0-35B (the 1.0 card calls it the '35B-MoE' member; same 64.2 / 75.6 / 50.4 cells as its own card).",
    "source_id": "card3-ornith-ai-ornith-1-5-35b-a3b"
  },
  {
    "model": "laguna-s-2-1",
    "bench": "terminal-bench-2.1",
    "score": 0.702,
    "source": "https://huggingface.co/poolside/Laguna-S-2.1/blob/0f573140834b11cfac0c2af97a101a7a69a13e22/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-07-21",
    "note": "Poolside Laguna S 2.1 model card (pinned 0f57314), 'Benchmark results' table, row 'Laguna S 2.1', column 'Terminal-Bench 2.1' = 70.2%. 'Benchmarks as of 21 July 2026.' Native reasoning model (interleaved thinking); harness not printed on this card (the family's XS 2.1 card: Harbor framework with Poolside's 'pool' agent). Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-s-2-1",
    "bench": "swe-bench-multilingual",
    "score": 0.785,
    "source": "https://huggingface.co/poolside/Laguna-S-2.1/blob/0f573140834b11cfac0c2af97a101a7a69a13e22/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-07-21",
    "note": "Poolside Laguna S 2.1 model card (pinned 0f57314), 'Benchmark results' table, row 'Laguna S 2.1', column 'SWE-bench Multilingual' = 78.5%. 'Benchmarks as of 21 July 2026.' Native reasoning model (interleaved thinking); harness not printed on this card (the family's XS 2.1 card: Harbor framework with Poolside's 'pool' agent). Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-s-2-1",
    "bench": "swe-bench-pro",
    "score": 0.594,
    "source": "https://huggingface.co/poolside/Laguna-S-2.1/blob/0f573140834b11cfac0c2af97a101a7a69a13e22/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-07-21",
    "note": "Poolside Laguna S 2.1 model card (pinned 0f57314), 'Benchmark results' table, row 'Laguna S 2.1', column 'SWE-Bench Pro (Public Dataset)' = 59.4%. 'Benchmarks as of 21 July 2026.' Native reasoning model (interleaved thinking); harness not printed on this card (the family's XS 2.1 card: Harbor framework with Poolside's 'pool' agent). Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-s-2-1",
    "bench": "toolathlon-verified",
    "score": 0.497,
    "source": "https://huggingface.co/poolside/Laguna-S-2.1/blob/0f573140834b11cfac0c2af97a101a7a69a13e22/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-07-21",
    "note": "Poolside Laguna S 2.1 model card (pinned 0f57314), 'Benchmark results' table, row 'Laguna S 2.1', column 'Toolathlon Verified' = 49.7%. 'Benchmarks as of 21 July 2026.' Native reasoning model (interleaved thinking); harness not printed on this card (the family's XS 2.1 card: Harbor framework with Poolside's 'pool' agent). Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2-1",
    "bench": "swe-bench-verified",
    "score": 0.709,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, row 'Laguna XS 2.1', column 'SWE-bench Verified' = 70.9%. Card methodology: 'All benchmarking for Laguna XS 2.1 was completed using Laude Institute's Harbor Framework with our agent harness, with a maximum of 500 steps and sandboxed execution ... temperature=1.0, top_k=20 and top_p=1, with thinking mode enabled and a context length of 256K tokens.' Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-verified",
    "score": 0.699,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, same-lab row 'Laguna XS.2' (bold, Poolside's own earlier model = catalogue laguna-xs-2, 33B), column 'SWE-bench Verified' = 69.9%. Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2-1",
    "bench": "swe-bench-multilingual",
    "score": 0.631,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, row 'Laguna XS 2.1', column 'SWE-bench Multilingual' = 63.1%. Card methodology: 'All benchmarking for Laguna XS 2.1 was completed using Laude Institute's Harbor Framework with our agent harness, with a maximum of 500 steps and sandboxed execution ... temperature=1.0, top_k=20 and top_p=1, with thinking mode enabled and a context length of 256K tokens.' Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-multilingual",
    "score": 0.577,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, same-lab row 'Laguna XS.2' (bold, Poolside's own earlier model = catalogue laguna-xs-2, 33B), column 'SWE-bench Multilingual' = 57.7%. Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2-1",
    "bench": "swe-bench-pro",
    "score": 0.476,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, row 'Laguna XS 2.1', column 'SWE-Bench Pro (Public Dataset)' = 47.6%. Card methodology: 'All benchmarking for Laguna XS 2.1 was completed using Laude Institute's Harbor Framework with our agent harness, with a maximum of 500 steps and sandboxed execution ... temperature=1.0, top_k=20 and top_p=1, with thinking mode enabled and a context length of 256K tokens.' Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2",
    "bench": "swe-bench-pro",
    "score": 0.463,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, same-lab row 'Laguna XS.2' (bold, Poolside's own earlier model = catalogue laguna-xs-2, 33B), column 'SWE-Bench Pro (Public Dataset)' = 46.3%. Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2-1",
    "bench": "terminal-bench-2",
    "score": 0.375,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, row 'Laguna XS 2.1', column 'Terminal-Bench 2.0' = 37.5%. Card methodology: 'All benchmarking for Laguna XS 2.1 was completed using Laude Institute's Harbor Framework with our agent harness, with a maximum of 500 steps and sandboxed execution ... temperature=1.0, top_k=20 and top_p=1, with thinking mode enabled and a context length of 256K tokens.' Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "laguna-xs-2",
    "bench": "terminal-bench-2",
    "score": 0.357,
    "source": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md#benchmark-results",
    "by": "lab",
    "date": "2026-06-20",
    "note": "Poolside Laguna XS 2.1 model card (pinned c5f3626), 'Benchmark results' table, same-lab row 'Laguna XS.2' (bold, Poolside's own earlier model = catalogue laguna-xs-2, 33B), column 'Terminal-Bench 2.0' = 35.7%. Vendor-reported (rule b): Poolside's own figure for its own model, hand-recorded; other labs' rows not used (card: comparison scores are 'official scores published in release blog posts or equivalent' or 'as reported by third parties').",
    "source_id": "first-party-poolside-scores"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu",
    "score": 0.9096,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'DeepSeek V3.1' = 90.96. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu",
    "score": 0.9023,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'Qwen3 MoE-2507' = 90.23. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "mmlu",
    "score": 0.8986,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'Kimi-K2' = 89.86. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "mmlu",
    "score": 0.8964,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'GPT-4.1' = 89.64. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "mmlu",
    "score": 0.9175,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'Claude4 Sonnet' = 91.75. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "mmlu",
    "score": 0.8633,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'Gemini2.5 Flash' = 86.33. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "mmlu",
    "score": 0.8971,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU' (acc), column 'LongCat-Flash' = 89.71. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-pro",
    "score": 0.8445,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'DeepSeek V3.1' = 84.45. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.8483,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Qwen3 MoE-2507' = 84.83. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "mmlu-pro",
    "score": 0.8206,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Kimi-K2' = 82.06. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "mmlu-pro",
    "score": 0.8172,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'GPT-4.1' = 81.72. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "mmlu-pro",
    "score": 0.8374,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Claude4 Sonnet' = 83.74. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "mmlu-pro",
    "score": 0.8195,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Gemini2.5 Flash' = 81.95. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "mmlu-pro",
    "score": 0.8268,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'LongCat-Flash' = 82.68. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "c-eval",
    "score": 0.8921,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'DeepSeek V3.1' = 89.21. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "c-eval",
    "score": 0.927,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'Qwen3 MoE-2507' = 92.70. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "c-eval",
    "score": 0.9126,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'Kimi-K2' = 91.26. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "c-eval",
    "score": 0.7953,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'GPT-4.1' = 79.53. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "c-eval",
    "score": 0.8663,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'Claude4 Sonnet' = 86.63. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "c-eval",
    "score": 0.7878,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'Gemini2.5 Flash' = 78.78. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "c-eval",
    "score": 0.9044,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CEval' (acc), column 'LongCat-Flash' = 90.44. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "cmmlu",
    "score": 0.8804,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'DeepSeek V3.1' = 88.04. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "cmmlu",
    "score": 0.8814,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'Qwen3 MoE-2507' = 88.14. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "cmmlu",
    "score": 0.8966,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'Kimi-K2' = 89.66. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "cmmlu",
    "score": 0.7765,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'GPT-4.1' = 77.65. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "cmmlu",
    "score": 0.8651,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'Claude4 Sonnet' = 86.51. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "cmmlu",
    "score": 0.783,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'Gemini2.5 Flash' = 78.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "cmmlu",
    "score": 0.8434,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'CMMLU' (acc), column 'LongCat-Flash' = 84.34. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "ifeval-unspecified",
    "score": 0.8669,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'DeepSeek V3.1' = 86.69. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.8854,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'Qwen3 MoE-2507' = 88.54. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "ifeval-unspecified",
    "score": 0.8891,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'Kimi-K2' = 88.91. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "ifeval-unspecified",
    "score": 0.8558,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'GPT-4.1' = 85.58. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "ifeval-unspecified",
    "score": 0.8835,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'Claude4 Sonnet' = 88.35. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "ifeval-unspecified",
    "score": 0.8392,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'Gemini2.5 Flash' = 83.92. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "ifeval-unspecified",
    "score": 0.8965,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (acc), column 'LongCat-Flash' = 89.65. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "collie",
    "score": 0.438,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'DeepSeek V3.1' = 43.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "collie",
    "score": 0.4971,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'Qwen3 MoE-2507' = 49.71. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "collie",
    "score": 0.5634,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'Kimi-K2' = 56.34. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "collie",
    "score": 0.5,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'GPT-4.1' = 50.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "collie",
    "score": 0.5122,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'Claude4 Sonnet' = 51.22. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "collie",
    "score": 0.486,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'Gemini2.5 Flash' = 48.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "collie",
    "score": 0.571,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'COLLIE' (acc), column 'LongCat-Flash' = 57.10. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "math-500",
    "score": 0.9608,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'DeepSeek V3.1' = 96.08. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "math-500",
    "score": 0.988,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'Qwen3 MoE-2507' = 98.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "math-500",
    "score": 0.976,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'Kimi-K2' = 97.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "math-500",
    "score": 0.906,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'GPT-4.1' = 90.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "math-500",
    "score": 0.938,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'Claude4 Sonnet' = 93.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "math-500",
    "score": 0.984,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'Gemini2.5 Flash' = 98.40. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "math-500",
    "score": 0.964,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (acc), column 'LongCat-Flash' = 96.40. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aime-2024",
    "score": 0.8167,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (avg@10), column 'Qwen3 MoE-2507' = 81.67. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "aime-2024",
    "score": 0.47,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (avg@10), column 'GPT-4.1' = 47.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "aime-2024",
    "score": 0.47,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (avg@10), column 'Claude4 Sonnet' = 47.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "aime-2024",
    "score": 0.7967,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (avg@10), column 'Gemini2.5 Flash' = 79.67. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "aime-2024",
    "score": 0.7042,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (avg@10), column 'LongCat-Flash' = 70.42. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aime-2025",
    "score": 0.4927,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'DeepSeek V3.1' = 49.27. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.6833,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'Qwen3 MoE-2507' = 68.33. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "aime-2025",
    "score": 0.5066,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'Kimi-K2' = 50.66. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "aime-2025",
    "score": 0.32,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'GPT-4.1' = 32.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "aime-2025",
    "score": 0.37,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'Claude4 Sonnet' = 37.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "aime-2025",
    "score": 0.6733,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'Gemini2.5 Flash' = 67.33. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "aime-2025",
    "score": 0.6125,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (avg@10), column 'LongCat-Flash' = 61.25. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "beyondaime",
    "score": 0.365,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'DeepSeek V3.1' = 36.50. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "beyondaime",
    "score": 0.576,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'Qwen3 MoE-2507' = 57.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "beyondaime",
    "score": 0.366,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'Kimi-K2' = 36.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "beyondaime",
    "score": 0.221,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'GPT-4.1' = 22.10. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "beyondaime",
    "score": 0.205,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'Claude4 Sonnet' = 20.50. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "beyondaime",
    "score": 0.442,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'Gemini2.5 Flash' = 44.20. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "beyondaime",
    "score": 0.43,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (avg@10), column 'LongCat-Flash' = 43.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "gpqa-diamond",
    "score": 0.7743,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'Qwen3 MoE-2507' = 77.43. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "gpqa-diamond",
    "score": 0.7576,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'Kimi-K2' = 75.76. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "gpqa-diamond",
    "score": 0.6768,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'GPT-4.1' = 67.68. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "gpqa-diamond",
    "score": 0.7071,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'Claude4 Sonnet' = 70.71. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "gpqa-diamond",
    "score": 0.803,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'Gemini2.5 Flash' = 80.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "gpqa-diamond",
    "score": 0.7323,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-diamond' (acc), column 'LongCat-Flash' = 73.23. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "drop",
    "score": 0.8419,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'DeepSeek V3.1' = 84.19. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "drop",
    "score": 0.7857,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'Qwen3 MoE-2507' = 78.57. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "drop",
    "score": 0.8904,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'Kimi-K2' = 89.04. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "drop",
    "score": 0.6694,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'GPT-4.1' = 66.94. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "drop",
    "score": 0.7306,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'Claude4 Sonnet' = 73.06. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "drop",
    "score": 0.4503,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'Gemini2.5 Flash' = 45.03. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "drop",
    "score": 0.7906,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'DROP' (f1), column 'LongCat-Flash' = 79.06. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "zebralogic",
    "score": 0.853,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'DeepSeek V3.1' = 85.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "zebralogic",
    "score": 0.9422,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'Qwen3 MoE-2507' = 94.22. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "zebralogic",
    "score": 0.8911,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'Kimi-K2' = 89.11. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "zebralogic",
    "score": 0.7585,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'Claude4 Sonnet' = 75.85. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "zebralogic",
    "score": 0.5178,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'Gemini2.5 Flash' = 51.78. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "zebralogic",
    "score": 0.893,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (acc), column 'LongCat-Flash' = 89.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "livecodebench-unversioned",
    "score": 0.4648,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'Qwen3 MoE-2507' = 46.48. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "livecodebench-unversioned",
    "score": 0.467,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'Kimi-K2' = 46.70. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "livecodebench-unversioned",
    "score": 0.3921,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'GPT-4.1' = 39.21. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "livecodebench-unversioned",
    "score": 0.4559,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'Claude4 Sonnet' = 45.59. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "livecodebench-unversioned",
    "score": 0.3965,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'Gemini2.5 Flash' = 39.65. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "livecodebench-unversioned",
    "score": 0.4802,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (pass@1), column 'LongCat-Flash' = 48.02. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "humaneval-plus",
    "score": 0.9268,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'DeepSeek V3.1' = 92.68. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "humaneval-plus",
    "score": 0.9451,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'Qwen3 MoE-2507' = 94.51. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "humaneval-plus",
    "score": 0.8598,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'Kimi-K2' = 85.98. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "humaneval-plus",
    "score": 0.9329,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'GPT-4.1' = 93.29. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "humaneval-plus",
    "score": 0.9451,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'Claude4 Sonnet' = 94.51. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "humaneval-plus",
    "score": 0.878,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'Gemini2.5 Flash' = 87.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "humaneval-plus",
    "score": 0.8841,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'Humaneval+' (pass@1), column 'LongCat-Flash' = 88.41. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mbpp-plus",
    "score": 0.7989,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'DeepSeek V3.1' = 79.89. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "mbpp-plus",
    "score": 0.7989,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'Qwen3 MoE-2507' = 79.89. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "mbpp-plus",
    "score": 0.8175,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'Kimi-K2' = 81.75. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "mbpp-plus",
    "score": 0.7937,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'GPT-4.1' = 79.37. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "mbpp-plus",
    "score": 0.8016,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'Claude4 Sonnet' = 80.16. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "mbpp-plus",
    "score": 0.7619,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'Gemini2.5 Flash' = 76.19. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "mbpp-plus",
    "score": 0.7963,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'MBPP+' (pass@1), column 'LongCat-Flash' = 79.63. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "swe-bench-verified",
    "score": 0.42,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'SWE-Bench-Verified' (acc), column 'Qwen3 MoE-2507' = 42.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "swe-bench-verified",
    "score": 0.646,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'SWE-Bench-Verified' (acc), column 'Kimi-K2' = 64.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "swe-bench-verified",
    "score": 0.486,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'SWE-Bench-Verified' (acc), column 'GPT-4.1' = 48.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "swe-bench-verified",
    "score": 0.406,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'SWE-Bench-Verified' (acc), column 'Gemini2.5 Flash' = 40.60. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "swe-bench-verified",
    "score": 0.604,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'SWE-Bench-Verified' (acc), column 'LongCat-Flash' = 60.40. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "terminal-bench-pre-2",
    "score": 0.1728,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'Qwen3 MoE-2507' = 17.28. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "terminal-bench-pre-2",
    "score": 0.2593,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'Kimi-K2' = 25.93. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "terminal-bench-pre-2",
    "score": 0.284,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'GPT-4.1' = 28.40. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "terminal-bench-pre-2",
    "score": 0.4074,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'Claude4 Sonnet' = 40.74. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "terminal-bench-pre-2",
    "score": 0.1235,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'Gemini2.5 Flash' = 12.35. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "terminal-bench-pre-2",
    "score": 0.3951,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'TerminalBench' (acc), column 'LongCat-Flash' = 39.51. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "tau2-telecom",
    "score": 0.7368,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (telecom)' (avg@4), column 'LongCat-Flash' = 73.68. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-airline",
    "score": 0.46,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'DeepSeek V3.1' = 46.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-airline",
    "score": 0.36,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'Qwen3 MoE-2507' = 36.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-airline",
    "score": 0.542,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'Kimi-K2' = 54.20. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-airline",
    "score": 0.56,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'GPT-4.1' = 56.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau2-airline",
    "score": 0.6,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'Claude4 Sonnet' = 60.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "tau2-airline",
    "score": 0.415,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'Gemini2.5 Flash' = 41.50. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (airline)' (avg@4), column 'LongCat-Flash' = 58.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-retail",
    "score": 0.649,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'DeepSeek V3.1' = 64.90. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "tau2-retail",
    "score": 0.705,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'Qwen3 MoE-2507' = 70.50. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "tau2-retail",
    "score": 0.708,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'Kimi-K2' = 70.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "tau2-retail",
    "score": 0.741,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'GPT-4.1' = 74.10. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "tau2-retail",
    "score": 0.8,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'Claude4 Sonnet' = 80.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "tau2-retail",
    "score": 0.648,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'Gemini2.5 Flash' = 64.80. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "tau2-retail",
    "score": 0.7127,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench (retail)' (avg@4), column 'LongCat-Flash' = 71.27. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "vita-bench",
    "score": 0.203,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'DeepSeek V3.1' = 20.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "vita-bench",
    "score": 0.085,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'Qwen3 MoE-2507' = 8.50. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking (Instruct-2507, card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "kimi-k2",
    "bench": "vita-bench",
    "score": 0.182,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'Kimi-K2' = 18.20. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. 1043B total / 32B active; card published (repo created 2025-08-29) before Kimi-K2-Instruct-0905, so the July 2025 Kimi-K2-Instruct, as for the imported τ² telecom row. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gpt-4-1",
    "bench": "vita-bench",
    "score": 0.19,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'GPT-4.1' = 19.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-reasoning model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "vita-bench",
    "score": 0.23,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'Claude4 Sonnet' = 23.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "vita-bench",
    "score": 0.08,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'Gemini2.5 Flash' = 8.00. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. non-thinking mode (card note). Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "vita-bench",
    "score": 0.243,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat/blob/f0b8e8d7380330f450ca1fde4ec56823a514584c/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Chat card (@f0b8e8d, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (avg@4), column 'LongCat-Flash' = 24.30. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: 'DeepSeek-V3.1, Qwen3-235B-A22B, Gemini2.5-Flash, and Claude4-Sonnet are evaluated under their non-thinking mode.' LongCat-Flash-Chat is a non-thinking model.",
    "source_id": "card-meituan-longcat-flash-chat"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-pro",
    "score": 0.844,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'DeepSeek-V3.1-Thinking' = 84.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-pro",
    "score": 0.844,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Qwen3-235B-A22B-Thinking-2507' = 84.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "mmlu-pro",
    "score": 0.815,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'GLM-4.5' = 81.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "mmlu-pro",
    "score": 0.853,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'OpenAI-o3' = 85.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "mmlu-pro",
    "score": 0.867,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'Gemini2.5-Pro' = 86.7. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "mmlu-pro",
    "score": 0.826,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Pro' (acc), column 'LongCat-Flash-Thinking' = 82.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "mmlu-redux",
    "score": 0.905,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'DeepSeek-V3.1-Thinking' = 90.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "mmlu-redux",
    "score": 0.914,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'Qwen3-235B-A22B-Thinking-2507' = 91.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "mmlu-redux",
    "score": 0.899,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'GLM-4.5' = 89.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "mmlu-redux",
    "score": 0.931,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'OpenAI-o3' = 93.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "mmlu-redux",
    "score": 0.901,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'Gemini2.5-Pro' = 90.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "mmlu-redux",
    "score": 0.893,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MMLU-Redux' (acc), column 'LongCat-Flash-Thinking' = 89.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "ifeval-prompt-strict",
    "score": 0.863,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'DeepSeek-V3.1-Thinking' = 86.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ifeval-prompt-strict",
    "score": 0.893,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'Qwen3-235B-A22B-Thinking-2507' = 89.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "ifeval-prompt-strict",
    "score": 0.854,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'GLM-4.5' = 85.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "ifeval-prompt-strict",
    "score": 0.902,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'OpenAI-o3' = 90.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "ifeval-prompt-strict",
    "score": 0.924,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'Gemini2.5-Pro' = 92.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "ifeval-prompt-strict",
    "score": 0.869,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'IFEval' (strict prompt), column 'LongCat-Flash-Thinking' = 86.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "math-500",
    "score": 0.988,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'DeepSeek-V3.1-Thinking' = 98.8. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "math-500",
    "score": 0.996,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'Qwen3-235B-A22B-Thinking-2507' = 99.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "math-500",
    "score": 0.954,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'GLM-4.5' = 95.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "math-500",
    "score": 0.984,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'OpenAI-o3' = 98.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "math-500",
    "score": 0.98,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'Gemini2.5-Pro' = 98.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "math-500",
    "score": 0.992,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MATH500' (Mean@1), column 'LongCat-Flash-Thinking' = 99.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.804,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'DeepSeek-V3.1-Thinking' = 80.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.838,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'Qwen3-235B-A22B-Thinking-2507' = 83.8. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.763,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'GLM-4.5' = 76.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.719,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'OpenAI-o3' = 71.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.793,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'Gemini2.5-Pro' = 79.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.837,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'HMMT25' (Mean@32), column 'LongCat-Flash-Thinking' = 83.7. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aime-2024",
    "score": 0.939,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (Mean@32), column 'DeepSeek-V3.1-Thinking' = 93.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "aime-2024",
    "score": 0.939,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (Mean@32), column 'Qwen3-235B-A22B-Thinking-2507' = 93.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2024",
    "score": 0.893,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (Mean@32), column 'GLM-4.5' = 89.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "aime-2024",
    "score": 0.907,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (Mean@32), column 'Gemini2.5-Pro' = 90.7. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "aime-2024",
    "score": 0.933,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME24' (Mean@32), column 'LongCat-Flash-Thinking' = 93.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "aime-2025",
    "score": 0.879,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (Mean@32), column 'DeepSeek-V3.1-Thinking' = 87.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "aime-2025",
    "score": 0.925,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (Mean@32), column 'Qwen3-235B-A22B-Thinking-2507' = 92.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "aime-2025",
    "score": 0.855,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (Mean@32), column 'GLM-4.5' = 85.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "aime-2025",
    "score": 0.892,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (Mean@32), column 'Gemini2.5-Pro' = 89.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "aime-2025",
    "score": 0.906,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'AIME25' (Mean@32), column 'LongCat-Flash-Thinking' = 90.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "beyondaime",
    "score": 0.718,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'DeepSeek-V3.1-Thinking' = 71.8. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "beyondaime",
    "score": 0.715,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'Qwen3-235B-A22B-Thinking-2507' = 71.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "beyondaime",
    "score": 0.66,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'GLM-4.5' = 66.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "beyondaime",
    "score": 0.632,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'OpenAI-o3' = 63.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "beyondaime",
    "score": 0.63,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'Gemini2.5-Pro' = 63.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "beyondaime",
    "score": 0.695,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BeyondAIME' (Mean@10), column 'LongCat-Flash-Thinking' = 69.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "gpqa-diamond",
    "score": 0.842,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'DeepSeek-V3.1-Thinking' = 84.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "gpqa-diamond",
    "score": 0.804,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'Qwen3-235B-A22B-Thinking-2507' = 80.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "gpqa-diamond",
    "score": 0.783,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'GLM-4.5' = 78.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "gpqa-diamond",
    "score": 0.819,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'OpenAI-o3' = 81.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "gpqa-diamond",
    "score": 0.84,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'Gemini2.5-Pro' = 84.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "gpqa-diamond",
    "score": 0.815,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'GPQA-Diamond' (Mean@16), column 'LongCat-Flash-Thinking' = 81.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "zebralogic",
    "score": 0.961,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'DeepSeek-V3.1-Thinking' = 96.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "zebralogic",
    "score": 0.975,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'Qwen3-235B-A22B-Thinking-2507' = 97.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "zebralogic",
    "score": 0.909,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'GLM-4.5' = 90.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "zebralogic",
    "score": 0.943,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'OpenAI-o3' = 94.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "zebralogic",
    "score": 0.924,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'Gemini2.5-Pro' = 92.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "zebralogic",
    "score": 0.955,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'ZebraLogic' (Mean@1), column 'LongCat-Flash-Thinking' = 95.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "livecodebench-unversioned",
    "score": 0.735,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'DeepSeek-V3.1-Thinking' = 73.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "livecodebench-unversioned",
    "score": 0.754,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'Qwen3-235B-A22B-Thinking-2507' = 75.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "livecodebench-unversioned",
    "score": 0.611,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'GLM-4.5' = 61.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "livecodebench-unversioned",
    "score": 0.762,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'OpenAI-o3' = 76.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "livecodebench-unversioned",
    "score": 0.742,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'Gemini2.5-Pro' = 74.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "livecodebench-unversioned",
    "score": 0.794,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'LiveCodeBench' (Mean@4), column 'LongCat-Flash-Thinking' = 79.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "ojbench",
    "score": 0.336,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'DeepSeek-V3.1-Thinking' = 33.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "ojbench",
    "score": 0.321,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'Qwen3-235B-A22B-Thinking-2507' = 32.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "ojbench",
    "score": 0.19,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'GLM-4.5' = 19.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "ojbench",
    "score": 0.384,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'OpenAI-o3' = 38.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "ojbench",
    "score": 0.416,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'Gemini2.5-Pro' = 41.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "ojbench",
    "score": 0.407,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'OJBench' (Mean@1), column 'LongCat-Flash-Thinking' = 40.7. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "bfcl-v3",
    "score": 0.554,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BFCL V3' (full), column 'DeepSeek-V3.1-Thinking' = 55.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "bfcl-v3",
    "score": 0.757,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BFCL V3' (full), column 'Qwen3-235B-A22B-Thinking-2507' = 75.7. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "bfcl-v3",
    "score": 0.791,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BFCL V3' (full), column 'GLM-4.5' = 79.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "bfcl-v3",
    "score": 0.632,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BFCL V3' (full), column 'Gemini2.5-Pro' = 63.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "bfcl-v3",
    "score": 0.744,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'BFCL V3' (full), column 'LongCat-Flash-Thinking' = 74.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-retail",
    "score": 0.654,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'DeepSeek-V3.1-Thinking' = 65.4. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-retail",
    "score": 0.682,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'Qwen3-235B-A22B-Thinking-2507' = 68.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "tau2-retail",
    "score": 0.693,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'GLM-4.5' = 69.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "tau2-retail",
    "score": 0.728,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'OpenAI-o3' = 72.8. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "tau2-retail",
    "score": 0.709,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'Gemini2.5-Pro' = 70.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "tau2-retail",
    "score": 0.715,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Retail' (Mean@4), column 'LongCat-Flash-Thinking' = 71.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "tau2-airline",
    "score": 0.44,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'DeepSeek-V3.1-Thinking' = 44.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'Qwen3-235B-A22B-Thinking-2507' = 58.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "tau2-airline",
    "score": 0.66,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'GLM-4.5' = 66.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "tau2-airline",
    "score": 0.625,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'OpenAI-o3' = 62.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'Gemini2.5-Pro' = 58.0. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "tau2-airline",
    "score": 0.675,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Airline' (Mean@4), column 'LongCat-Flash-Thinking' = 67.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "tau2-telecom",
    "score": 0.831,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'τ²-Bench-Telecom' (Mean@4), column 'LongCat-Flash-Thinking' = 83.1. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "vita-bench",
    "score": 0.135,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'DeepSeek-V3.1-Thinking' = 13.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "vita-bench",
    "score": 0.215,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'Qwen3-235B-A22B-Thinking-2507' = 21.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "vita-bench",
    "score": 0.268,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'GLM-4.5' = 26.8. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "vita-bench",
    "score": 0.353,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'OpenAI-o3' = 35.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "vita-bench",
    "score": 0.243,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'Gemini2.5-Pro' = 24.3. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "vita-bench",
    "score": 0.295,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'VitaBench' (), column 'LongCat-Flash-Thinking' = 29.5. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "deepseek-v3-1",
    "bench": "minif2f-pass1",
    "score": 0.496,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'DeepSeek-V3.1-Thinking' = 49.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. thinking mode. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "minif2f-pass1",
    "score": 0.119,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'Qwen3-235B-A22B-Thinking-2507' = 11.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "glm-4-5",
    "bench": "minif2f-pass1",
    "score": 0.109,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'GLM-4.5' = 10.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "o3",
    "bench": "minif2f-pass1",
    "score": 0.152,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'OpenAI-o3' = 15.2. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. reasoning effort not printed. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "minif2f-pass1",
    "score": 0.139,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "independent",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'Gemini2.5-Pro' = 13.9. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own run of another lab's model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-thinking",
    "bench": "minif2f-pass1",
    "score": 0.676,
    "source": "https://huggingface.co/meituan-longcat/LongCat-Flash-Thinking/blob/7c9aad1e2daceaeab3a5a03deedb8e11e7a7819f/README.md",
    "by": "lab",
    "date": "2025-09-24",
    "note": "Meituan LongCat-Flash-Thinking card (@7c9aad1, revision 2025-09-24), 'Evaluation Results' table, row 'MiniF2F-Test' (Pass@1), column 'LongCat-Flash-Thinking' = 67.6. Card note: 'Values marked with * are sourced from other public reports' - this cell is unstarred, so it is Meituan's own figure for its own model. Card note: LongCat-Flash-Thinking inference 'temperature=1.0, topk=-1, and topp=0.95'. GPT-5-Thinking column not used (reasoning effort unstated).",
    "source_id": "card-meituan-longcat-flash-thinking"
  },
  {
    "model": "longcat-flash-chat",
    "bench": "lmarena-text",
    "score": 1401.2,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "text_style_control, category overall, split latest; exact entry 'longcat-flash-chat' (organization meituan, license MIT); 11,123 votes; published 95% interval 1394.8–1407.6 (rating 1401.177). Same pinned October revision as the other lmarena-text rows; read with pyarrow 2026-10-10. Not the separately catalogued 'longcat-flash-chat-2602-exp' (Proprietary).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "magistral-medium-2509",
    "bench": "aime-2024",
    "score": 0.9182,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.2', column 'AIME24 pass@1' = 91.82%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.2 = API magistral-medium-2509 (version 25.09) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2509",
    "bench": "aime-2025",
    "score": 0.8348,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.2', column 'AIME25 pass@1' = 83.48%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.2 = API magistral-medium-2509 (version 25.09) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2509",
    "bench": "gpqa-diamond",
    "score": 0.7626,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.2', column 'GPQA Diamond' = 76.26%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.2 = API magistral-medium-2509 (version 25.09) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2509",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.75,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.2', column 'Livecodebench (v5)' = 75.00%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.2 = API magistral-medium-2509 (version 25.09) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2507",
    "bench": "aime-2024",
    "score": 0.7203,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.1', column 'AIME24 pass@1' = 72.03%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.1 = API magistral-medium-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2507",
    "bench": "aime-2025",
    "score": 0.6099,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.1', column 'AIME25 pass@1' = 60.99%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.1 = API magistral-medium-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2507",
    "bench": "gpqa-diamond",
    "score": 0.7146,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.1', column 'GPQA Diamond' = 71.46%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.1 = API magistral-medium-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-medium-2507",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.5935,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Medium 1.1', column 'Livecodebench (v5)' = 59.35%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Medium 1.1 = API magistral-medium-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2507",
    "bench": "aime-2024",
    "score": 0.7052,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.1', column 'AIME24 pass@1' = 70.52%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.1 = mistralai/Magistral-Small-2507 = API magistral-small-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2507",
    "bench": "aime-2025",
    "score": 0.6203,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.1', column 'AIME25 pass@1' = 62.03%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.1 = mistralai/Magistral-Small-2507 = API magistral-small-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2507",
    "bench": "gpqa-diamond",
    "score": 0.6578,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.1', column 'GPQA Diamond' = 65.78%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.1 = mistralai/Magistral-Small-2507 = API magistral-small-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "magistral-small-2507",
    "bench": "livecodebench-v5-window-unspecified",
    "score": 0.5917,
    "source": "https://huggingface.co/mistralai/Magistral-Small-2509/blob/a31cc96ab10cf19bc42c628fedf1e359e0853c49/README.md",
    "by": "lab",
    "date": "2025-09-17",
    "note": "Magistral-Small-2509 model card (Apache-2.0, pinned a31cc96), '## Benchmark Results' table, row 'Magistral Small 1.1', column 'Livecodebench (v5)' = 59.17%. Same-lab row in Mistral's own table (no cell marked as copied). Card sampling parameters: top_p 0.95, temperature 0.7, max_tokens 131072. Checkpoint: Magistral Small 1.1 = mistralai/Magistral-Small-2507 = API magistral-small-2507 (version 25.07) (Mistral docs model card). Same column as the existing Magistral Small 1.2 / Medium 1.0 / Small 1.0 rows; LiveCodeBench v5 window not stated.",
    "source_id": "card2-mistralai-magistral-small-2509"
  },
  {
    "model": "minicpm5-2b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.691,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'MiniCPM5-2B' = 69.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.421,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'LFM2.5-2.6B' = 42.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.202,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'Qwen3.5-2B' = 20.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.429,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'Gemma-4-E2B-it' = 42.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.564,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'Qwen3.5-4B' = 56.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.589,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'granite-4.2-3B' = 58.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.507,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'Nemotron-3-Nano-4B' = 50.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.539,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'Gemma-4-E4B-it' = 53.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.398,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LiveCodeBench v6', column 'LFM2.5-8B-A1B' = 39.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.68,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'MiniCPM5-2B' = 68.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.309,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'LFM2.5-2.6B' = 30.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.103,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'Qwen3.5-2B' = 10.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.271,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'Gemma-4-E2B-it' = 27.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.583,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'Qwen3.5-4B' = 58.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.546,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'granite-4.2-3B' = 54.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.516,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'Nemotron-3-Nano-4B' = 51.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.458,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'Gemma-4-E4B-it' = 45.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.278,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Easy)', column 'LFM2.5-8B-A1B' = 27.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.175,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'MiniCPM5-2B' = 17.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'LFM2.5-2.6B' = 0.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'Qwen3.5-2B' = 0.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'Gemma-4-E2B-it' = 0.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.07,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'Qwen3.5-4B' = 7.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.053,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'granite-4.2-3B' = 5.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.053,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'Nemotron-3-Nano-4B' = 5.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.018,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'Gemma-4-E4B-it' = 1.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LCB-Pro 25Q2 (Medium)', column 'LFM2.5-8B-A1B' = 0.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "ojbench",
    "score": 0.325,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'MiniCPM5-2B' = 32.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "ojbench",
    "score": 0.112,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'LFM2.5-2.6B' = 11.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ojbench",
    "score": 0.026,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'Qwen3.5-2B' = 2.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ojbench",
    "score": 0.116,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'Gemma-4-E2B-it' = 11.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ojbench",
    "score": 0.248,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'Qwen3.5-4B' = 24.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ojbench",
    "score": 0.218,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'granite-4.2-3B' = 21.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "ojbench",
    "score": 0.2,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'Nemotron-3-Nano-4B' = 20.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ojbench",
    "score": 0.19,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'Gemma-4-E4B-it' = 19.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "ojbench",
    "score": 0.082,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'OJBench', column 'LFM2.5-8B-A1B' = 8.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "aime-2025",
    "score": 0.865,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'MiniCPM5-2B' = 86.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "aime-2025",
    "score": 0.419,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'LFM2.5-2.6B' = 41.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "aime-2025",
    "score": 0.296,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'Qwen3.5-2B' = 29.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "aime-2025",
    "score": 0.317,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'Gemma-4-E2B-it' = 31.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "aime-2025",
    "score": 0.788,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'Qwen3.5-4B' = 78.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "aime-2025",
    "score": 0.794,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'granite-4.2-3B' = 79.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "aime-2025",
    "score": 0.563,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'Nemotron-3-Nano-4B' = 56.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "aime-2025",
    "score": 0.371,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'Gemma-4-E4B-it' = 37.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "aime-2025",
    "score": 0.46,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2025', column 'LFM2.5-8B-A1B' = 46.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "aime-2026",
    "score": 0.865,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'MiniCPM5-2B' = 86.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "aime-2026",
    "score": 0.452,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'LFM2.5-2.6B' = 45.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "aime-2026",
    "score": 0.29,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'Qwen3.5-2B' = 29.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "aime-2026",
    "score": 0.398,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'Gemma-4-E2B-it' = 39.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "aime-2026",
    "score": 0.827,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'Qwen3.5-4B' = 82.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "aime-2026",
    "score": 0.835,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'granite-4.2-3B' = 83.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "aime-2026",
    "score": 0.621,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'Nemotron-3-Nano-4B' = 62.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "aime-2026",
    "score": 0.45,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'Gemma-4-E4B-it' = 45.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "aime-2026",
    "score": 0.567,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'AIME 2026', column 'LFM2.5-8B-A1B' = 56.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "hmmt-feb-2026",
    "score": 0.638,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'MiniCPM5-2B' = 63.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "hmmt-feb-2026",
    "score": 0.337,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'LFM2.5-2.6B' = 33.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hmmt-feb-2026",
    "score": 0.205,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'Qwen3.5-2B' = 20.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "hmmt-feb-2026",
    "score": 0.178,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'Gemma-4-E2B-it' = 17.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hmmt-feb-2026",
    "score": 0.64,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'Qwen3.5-4B' = 64.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "hmmt-feb-2026",
    "score": 0.608,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'granite-4.2-3B' = 60.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "hmmt-feb-2026",
    "score": 0.513,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'Nemotron-3-Nano-4B' = 51.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "hmmt-feb-2026",
    "score": 0.301,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'Gemma-4-E4B-it' = 30.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "hmmt-feb-2026",
    "score": 0.385,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'HMMT Feb 2026', column 'LFM2.5-8B-A1B' = 38.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "math-500",
    "score": 0.946,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'MiniCPM5-2B' = 94.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "math-500",
    "score": 0.896,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'LFM2.5-2.6B' = 89.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "math-500",
    "score": 0.858,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'Qwen3.5-2B' = 85.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "math-500",
    "score": 0.854,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'Gemma-4-E2B-it' = 85.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "math-500",
    "score": 0.99,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'Qwen3.5-4B' = 99.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "math-500",
    "score": 0.97,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'granite-4.2-3B' = 97.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "math-500",
    "score": 0.916,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'Nemotron-3-Nano-4B' = 91.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "math-500",
    "score": 0.882,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'Gemma-4-E4B-it' = 88.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "math-500",
    "score": 0.932,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MATH-500', column 'LFM2.5-8B-A1B' = 93.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "ifbench-unspecified",
    "score": 0.663,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'MiniCPM5-2B' = 66.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "ifbench-unspecified",
    "score": 0.59,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'LFM2.5-2.6B' = 59.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ifbench-unspecified",
    "score": 0.46,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'Qwen3.5-2B' = 46.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifbench-unspecified",
    "score": 0.257,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'Gemma-4-E2B-it' = 25.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifbench-unspecified",
    "score": 0.59,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'Qwen3.5-4B' = 59.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ifbench-unspecified",
    "score": 0.73,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'granite-4.2-3B' = 73.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "ifbench-unspecified",
    "score": 0.583,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'Nemotron-3-Nano-4B' = 58.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifbench-unspecified",
    "score": 0.283,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'Gemma-4-E4B-it' = 28.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "ifbench-unspecified",
    "score": 0.51,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFBench', column 'LFM2.5-8B-A1B' = 51.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "ifeval-unspecified",
    "score": 0.867,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'MiniCPM5-2B' = 86.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "ifeval-unspecified",
    "score": 0.934,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'LFM2.5-2.6B' = 93.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ifeval-unspecified",
    "score": 0.775,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'Qwen3.5-2B' = 77.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifeval-unspecified",
    "score": 0.314,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'Gemma-4-E2B-it' = 31.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifeval-unspecified",
    "score": 0.902,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'Qwen3.5-4B' = 90.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "ifeval-unspecified",
    "score": 0.937,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'granite-4.2-3B' = 93.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "ifeval-unspecified",
    "score": 0.88,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'Nemotron-3-Nano-4B' = 88.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifeval-unspecified",
    "score": 0.444,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'Gemma-4-E4B-it' = 44.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "ifeval-unspecified",
    "score": 0.908,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'IFEval', column 'LFM2.5-8B-A1B' = 90.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "multi-if-unspecified",
    "score": 0.718,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'MiniCPM5-2B' = 71.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "multi-if-unspecified",
    "score": 0.768,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'LFM2.5-2.6B' = 76.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "multi-if-unspecified",
    "score": 0.571,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'Qwen3.5-2B' = 57.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "multi-if-unspecified",
    "score": 0.403,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'Gemma-4-E2B-it' = 40.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "multi-if-unspecified",
    "score": 0.736,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'Qwen3.5-4B' = 73.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "multi-if-unspecified",
    "score": 0.759,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'granite-4.2-3B' = 75.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "multi-if-unspecified",
    "score": 0.659,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'Nemotron-3-Nano-4B' = 65.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "multi-if-unspecified",
    "score": 0.459,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'Gemma-4-E4B-it' = 45.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "multi-if-unspecified",
    "score": 0.714,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'Multi-IF', column 'LFM2.5-8B-A1B' = 71.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "mmlu-pro",
    "score": 0.708,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'MiniCPM5-2B' = 70.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "mmlu-pro",
    "score": 0.652,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'LFM2.5-2.6B' = 65.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-pro",
    "score": 0.643,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'Qwen3.5-2B' = 64.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmlu-pro",
    "score": 0.56,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'Gemma-4-E2B-it' = 56.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-pro",
    "score": 0.78,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'Qwen3.5-4B' = 78.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "mmlu-pro",
    "score": 0.658,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'granite-4.2-3B' = 65.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "mmlu-pro",
    "score": 0.657,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'Nemotron-3-Nano-4B' = 65.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmlu-pro",
    "score": 0.683,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'Gemma-4-E4B-it' = 68.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "mmlu-pro",
    "score": 0.631,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Pro', column 'LFM2.5-8B-A1B' = 63.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "mmlu-redux",
    "score": 0.847,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'MiniCPM5-2B' = 84.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "mmlu-redux",
    "score": 0.8,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'LFM2.5-2.6B' = 80.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-redux",
    "score": 0.8,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'Qwen3.5-2B' = 80.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmlu-redux",
    "score": 0.718,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'Gemma-4-E2B-it' = 71.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-redux",
    "score": 0.887,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'Qwen3.5-4B' = 88.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "mmlu-redux",
    "score": 0.789,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'granite-4.2-3B' = 78.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "mmlu-redux",
    "score": 0.798,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'Nemotron-3-Nano-4B' = 79.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmlu-redux",
    "score": 0.837,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'Gemma-4-E4B-it' = 83.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "mmlu-redux",
    "score": 0.8,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'MMLU-Redux', column 'LFM2.5-8B-A1B' = 80.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "supergpqa",
    "score": 0.408,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'MiniCPM5-2B' = 40.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "supergpqa",
    "score": 0.262,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'LFM2.5-2.6B' = 26.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "supergpqa",
    "score": 0.386,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'Qwen3.5-2B' = 38.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "supergpqa",
    "score": 0.303,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'Gemma-4-E2B-it' = 30.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "supergpqa",
    "score": 0.528,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'Qwen3.5-4B' = 52.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "supergpqa",
    "score": 0.399,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'granite-4.2-3B' = 39.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "supergpqa",
    "score": 0.378,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'Nemotron-3-Nano-4B' = 37.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "supergpqa",
    "score": 0.387,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'Gemma-4-E4B-it' = 38.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "supergpqa",
    "score": 0.345,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SuperGPQA', column 'LFM2.5-8B-A1B' = 34.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "longbench-v2",
    "score": 0.437,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'MiniCPM5-2B' = 43.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "longbench-v2",
    "score": 0.303,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'LFM2.5-2.6B' = 30.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "longbench-v2",
    "score": 0.249,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'Qwen3.5-2B' = 24.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "longbench-v2",
    "score": 0.332,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'Gemma-4-E2B-it' = 33.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "longbench-v2",
    "score": 0.473,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'Qwen3.5-4B' = 47.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "longbench-v2",
    "score": 0.36,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'granite-4.2-3B' = 36.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "longbench-v2",
    "score": 0.32,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'Nemotron-3-Nano-4B' = 32.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "longbench-v2",
    "score": 0.427,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'Gemma-4-E4B-it' = 42.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "longbench-v2",
    "score": 0.304,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'LongBench v2', column 'LFM2.5-8B-A1B' = 30.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "tau3-banking",
    "score": 0.021,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ³-Bench Banking', column 'Qwen3.5-2B' = 2.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau3-banking",
    "score": 0.039,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ³-Bench Banking', column 'Gemma-4-E2B-it' = 3.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "tau3-banking",
    "score": 0.012,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ³-Bench Banking', column 'Nemotron-3-Nano-4B' = 1.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau3-banking",
    "score": 0.041,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ³-Bench Banking', column 'Gemma-4-E4B-it' = 4.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "tau3-banking",
    "score": 0.034,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ³-Bench Banking', column 'LFM2.5-8B-A1B' = 3.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "tau2-telecom",
    "score": 0.971,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ²-Bench Telecom', column 'MiniCPM5-2B' = 97.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "tau2-telecom",
    "score": 0.904,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ²-Bench Telecom', column 'LFM2.5-2.6B' = 90.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "tau2-telecom",
    "score": 0.409,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'τ²-Bench Telecom', column 'granite-4.2-3B' = 40.9. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "bfcl-v4",
    "score": 0.666,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'MiniCPM5-2B' = 66.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "bfcl-v4",
    "score": 0.611,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'LFM2.5-2.6B' = 61.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "bfcl-v4",
    "score": 0.436,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'Qwen3.5-2B' = 43.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "bfcl-v4",
    "score": 0.366,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'Gemma-4-E2B-it' = 36.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v4",
    "score": 0.568,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'Qwen3.5-4B' = 56.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "bfcl-v4",
    "score": 0.522,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'granite-4.2-3B' = 52.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "bfcl-v4",
    "score": 0.437,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'Nemotron-3-Nano-4B' = 43.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bfcl-v4",
    "score": 0.47,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'Gemma-4-E4B-it' = 47.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "bfcl-v4",
    "score": 0.492,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BFCL v4', column 'LFM2.5-8B-A1B' = 49.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "swe-bench-verified",
    "score": 0.464,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'MiniCPM5-2B' = 46.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "swe-bench-verified",
    "score": 0.06,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'LFM2.5-2.6B' = 6.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "swe-bench-verified",
    "score": 0.05,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'Qwen3.5-2B' = 5.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "swe-bench-verified",
    "score": 0.02,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'Gemma-4-E2B-it' = 2.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "swe-bench-verified",
    "score": 0.336,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'Qwen3.5-4B' = 33.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "swe-bench-verified",
    "score": 0.368,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'granite-4.2-3B' = 36.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "swe-bench-verified",
    "score": 0.03,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'Nemotron-3-Nano-4B' = 3.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "swe-bench-verified",
    "score": 0.15,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'Gemma-4-E4B-it' = 15.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "swe-bench-verified",
    "score": 0.004,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Verified', column 'LFM2.5-8B-A1B' = 0.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.144,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'MiniCPM5-2B' = 14.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.006,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'LFM2.5-2.6B' = 0.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.008,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'Qwen3.5-2B' = 0.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'Gemma-4-E2B-it' = 0.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.282,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'Qwen3.5-4B' = 28.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.123,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'granite-4.2-3B' = 12.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.001,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'Nemotron-3-Nano-4B' = 0.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.033,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'Gemma-4-E4B-it' = 3.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.004,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'SWE-bench Pro', column 'LFM2.5-8B-A1B' = 0.4. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "minicpm5-2b",
    "bench": "browsecomp-zh",
    "score": 0.435,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "lab",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'MiniCPM5-2B' = 43.5. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own figure for its own model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "browsecomp-zh",
    "score": 0.098,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'LFM2.5-2.6B' = 9.8. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "browsecomp-zh",
    "score": 0.182,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'Qwen3.5-2B' = 18.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "browsecomp-zh",
    "score": 0.047,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'Gemma-4-E2B-it' = 4.7. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "browsecomp-zh",
    "score": 0.396,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'Qwen3.5-4B' = 39.6. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "granite-4-2-3b",
    "bench": "browsecomp-zh",
    "score": 0.211,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'granite-4.2-3B' = 21.1. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "nemotron-3-nano-4b",
    "bench": "browsecomp-zh",
    "score": 0.033,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'Nemotron-3-Nano-4B' = 3.3. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "browsecomp-zh",
    "score": 0.07,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'Gemma-4-E4B-it' = 7.0. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "lfm2-5-8b-a1b",
    "bench": "browsecomp-zh",
    "score": 0.132,
    "source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/f97400052a43d642bbc6e9975e2397e3ae6a6b52/README.md",
    "by": "independent",
    "date": "2026-09-06",
    "note": "OpenBMB MiniCPM5-2B model card (Apache-2.0, pinned f974000), 'Evaluation Results of MiniCPM5-2B and Baselines' table, row 'BrowseComp-ZH', column 'LFM2.5-8B-A1B' = 13.2. Footnote 2: 'Scores marked † come from the official Artificial Analysis release; all others are reproduced internally' - this cell is unmarked, so OpenBMB's own run of another lab's model. Reasoning/thinking setting per model not printed; SWE-bench Pro split not stated; LiveCodeBench v6 window not stated.",
    "source_id": "card3-openbmb-minicpm5-2b"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmmu",
    "score": 0.645,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU val', column 'Qwen2-VL 72B' = 64.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mmmu",
    "score": 0.702,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU val', column 'Qwen2.5-VL 72B' = 70.2. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "mmmu",
    "score": 0.586,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU val', column 'Qwen2.5-VL 7B' = 58.6. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "mmmu",
    "score": 0.531,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU val', column 'Qwen2.5-VL 3B' = 53.1. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmmu-pro",
    "score": 0.462,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU-Pro overall', column 'Qwen2-VL 72B' = 46.2. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mmmu-pro",
    "score": 0.511,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU-Pro overall', column 'Qwen2.5-VL 72B' = 51.1. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "mmmu-pro",
    "score": 0.383,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU-Pro overall', column 'Qwen2.5-VL 7B' = 38.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "mmmu-pro",
    "score": 0.3156,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMMU-Pro overall', column 'Qwen2.5-VL 3B' = 31.56. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mathvista-mini",
    "score": 0.705,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MathVista mini', column 'Qwen2-VL 72B' = 70.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mathvista-mini",
    "score": 0.748,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MathVista mini', column 'Qwen2.5-VL 72B' = 74.8. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "mathvista-mini",
    "score": 0.682,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MathVista mini', column 'Qwen2.5-VL 7B' = 68.2. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "mathvista-mini",
    "score": 0.623,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MathVista mini', column 'Qwen2.5-VL 3B' = 62.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mmstar",
    "score": 0.683,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMStar', column 'Qwen2-VL 72B' = 68.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "mmstar",
    "score": 0.708,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMStar', column 'Qwen2.5-VL 72B' = 70.8. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "mmstar",
    "score": 0.639,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMStar', column 'Qwen2.5-VL 7B' = 63.9. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "mmstar",
    "score": 0.559,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'MMStar', column 'Qwen2.5-VL 3B' = 55.9. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "hallusionbench",
    "score": 0.581,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'HallBench avg', column 'Qwen2-VL 72B' = 58.1. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "hallusionbench",
    "score": 0.552,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'HallBench avg', column 'Qwen2.5-VL 72B' = 55.2. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "hallusionbench",
    "score": 0.529,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'HallBench avg', column 'Qwen2.5-VL 7B' = 52.9. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "hallusionbench",
    "score": 0.463,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'HallBench avg', column 'Qwen2.5-VL 3B' = 46.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "realworldqa",
    "score": 0.778,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'RealWorldQA avg', column 'Qwen2-VL 72B' = 77.8. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "realworldqa",
    "score": 0.757,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'RealWorldQA avg', column 'Qwen2.5-VL 72B' = 75.7. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "realworldqa",
    "score": 0.685,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'RealWorldQA avg', column 'Qwen2.5-VL 7B' = 68.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "realworldqa",
    "score": 0.654,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 3 (general visual benchmarks), row 'RealWorldQA avg', column 'Qwen2.5-VL 3B' = 65.4. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "cc-ocr",
    "score": 0.798,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'CC-OCR', column 'Qwen2.5-VL 72B' = 79.8. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "cc-ocr",
    "score": 0.778,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'CC-OCR', column 'Qwen2.5-VL 7B' = 77.8. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "cc-ocr",
    "score": 0.745,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'CC-OCR', column 'Qwen2.5-VL 3B' = 74.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "textvqa",
    "score": 0.835,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'TextVQA val', column 'Qwen2.5-VL 72B' = 83.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "textvqa",
    "score": 0.849,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'TextVQA val', column 'Qwen2.5-VL 7B' = 84.9. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "textvqa",
    "score": 0.793,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'TextVQA val', column 'Qwen2.5-VL 3B' = 79.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "docvqa",
    "score": 0.964,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'DocVQA test', column 'Qwen2.5-VL 72B' = 96.4. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "docvqa",
    "score": 0.957,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'DocVQA test', column 'Qwen2.5-VL 7B' = 95.7. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "docvqa",
    "score": 0.939,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'DocVQA test', column 'Qwen2.5-VL 3B' = 93.9. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "chartqa",
    "score": 0.895,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'ChartQA test Avg.', column 'Qwen2.5-VL 72B' = 89.5. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "chartqa",
    "score": 0.873,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'ChartQA test Avg.', column 'Qwen2.5-VL 7B' = 87.3. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "chartqa",
    "score": 0.84,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'ChartQA test Avg.', column 'Qwen2.5-VL 3B' = 84.0. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "ocrbench",
    "score": 0.885,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'OCRBench (0-1000 scale, divided by 10)', column 'Qwen2.5-VL 72B' = 885. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "ocrbench",
    "score": 0.864,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'OCRBench (0-1000 scale, divided by 10)', column 'Qwen2.5-VL 7B' = 864. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "ocrbench",
    "score": 0.797,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 5 (OCR, chart and document understanding), row 'OCRBench (0-1000 scale, divided by 10)', column 'Qwen2.5-VL 3B' = 797. Instruct checkpoints (the report's released models); other makers' columns not used. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "charxiv-rq",
    "score": 0.425,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1, Table 5 (OCR, chart and document understanding), row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 7B' = 42.5/73.9 (RQ = 42.5). Same table and row as the existing Qwen2.5-VL 72B CharXiv rows. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "charxiv-dq",
    "score": 0.739,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1, Table 5 (OCR, chart and document understanding), row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 7B' = 42.5/73.9 (DQ = 73.9). Same table and row as the existing Qwen2.5-VL 72B CharXiv rows. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "charxiv-rq",
    "score": 0.313,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1, Table 5 (OCR, chart and document understanding), row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 3B' = 31.3/58.6 (RQ = 31.3). Same table and row as the existing Qwen2.5-VL 72B CharXiv rows. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "charxiv-dq",
    "score": 0.586,
    "source": "https://arxiv.org/html/2502.13923v1",
    "by": "lab",
    "date": "2025-02-19",
    "note": "Qwen2.5-VL Technical Report, arXiv 2502.13923v1, Table 5 (OCR, chart and document understanding), row 'CharXiv RQ/DQ', column 'Qwen2.5-VL 3B' = 31.3/58.6 (DQ = 58.6). Same table and row as the existing Qwen2.5-VL 72B CharXiv rows. Vendor-reported (rule b).",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "solar-open-2",
    "bench": "mmlu-pro",
    "score": 0.862,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'Solar Open 2 / 250B-A15B' = 86.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "mmlu-pro",
    "score": 0.804,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MMLU-Pro', column 'Solar Open 1 / 102B-A12B' = 80.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "gpqa-diamond",
    "score": 0.863,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'Solar Open 2 / 250B-A15B' = 86.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "gpqa-diamond",
    "score": 0.662,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'GPQA-Diamond', column 'Solar Open 1 / 102B-A12B' = 66.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "hle",
    "score": 0.288,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'Solar Open 2 / 250B-A15B' = 28.8. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "hle",
    "score": 0.115,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HLE (w/o tools)', column 'Solar Open 1 / 102B-A12B' = 11.5. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.924,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'Solar Open 2 / 250B-A15B' = 92.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.565,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'LiveCodeBench (v6)', column 'Solar Open 1 / 102B-A12B' = 56.5. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "hmmt-feb-2026",
    "score": 0.939,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'Solar Open 2 / 250B-A15B' = 93.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "hmmt-feb-2026",
    "score": 0.689,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'HMMT2602', column 'Solar Open 1 / 102B-A12B' = 68.9. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "aime-2026",
    "score": 0.957,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'Solar Open 2 / 250B-A15B' = 95.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "aime-2026",
    "score": 0.877,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'AIME2026', column 'Solar Open 1 / 102B-A12B' = 87.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "multichallenge",
    "score": 0.61,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'Solar Open 2 / 250B-A15B' = 61.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "multichallenge",
    "score": 0.405,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Multi-Challenge', column 'Solar Open 1 / 102B-A12B' = 40.5. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "ifbench-unspecified",
    "score": 0.8,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'Solar Open 2 / 250B-A15B' = 80.0. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "ifbench-unspecified",
    "score": 0.577,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'IFBench', column 'Solar Open 1 / 102B-A12B' = 57.7. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "swe-bench-verified",
    "score": 0.704,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'Solar Open 2 / 250B-A15B' = 70.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "swe-bench-verified",
    "score": 0.154,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'SWE-Bench Verified', column 'Solar Open 1 / 102B-A12B' = 15.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.283,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'Solar Open 2 / 250B-A15B' = 28.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.023,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'Terminal Bench Hard', column 'Solar Open 1 / 102B-A12B' = 2.3. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "apex-agents",
    "score": 0.166,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'Solar Open 2 / 250B-A15B' = 16.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "apex-agents",
    "score": 0.024,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'APEX-Agents', column 'Solar Open 1 / 102B-A12B' = 2.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "mcp-atlas",
    "score": 0.582,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'Solar Open 2 / 250B-A15B' = 58.2. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "mcp-atlas",
    "score": 0.344,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'MCP-Atlas', column 'Solar Open 1 / 102B-A12B' = 34.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-2",
    "bench": "tau3-banking",
    "score": 0.196,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'Solar Open 2 / 250B-A15B' = 19.6. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: upstage/Solar-Open2-250B (the report's subject; no reasoning setting printed in the column header; the card recommends reasoning_effort high for reasoning). Table identical in v1 and v2.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "tau3-banking",
    "score": 0.074,
    "source": "https://arxiv.org/html/2607.20062v2#S5.T3",
    "by": "lab",
    "date": "2026-07-24",
    "note": "Upstage, Solar Open 2 Technical Report (arXiv 2607.20062v2, CC BY 4.0), Table 3 'English benchmark performance', row 'τ3 (banking)', column 'Solar Open 1 / 102B-A12B' = 7.4. Caption and Sec. 5.1: 'All numbers are measured with our internal evaluation harness under identical per-benchmark settings; reasoning-effort settings are noted in the table headers where applicable.' Upstage's own model (by lab); same table and settings as the four existing independent rows of this run. Checkpoint: Solar Open 1 = upstage/Solar-Open-100B (abstract: 'scaled up from Solar Open 1 (Solar Open 100B)'); same-lab previous-generation column. No reasoning setting printed.",
    "source_id": "paper-2607.20062v2-solar-open-2"
  },
  {
    "model": "solar-open-100b",
    "bench": "tau2-telecom",
    "score": 0.556,
    "source": "https://arxiv.org/html/2601.07022v1",
    "by": "lab",
    "date": "2026-01-11",
    "note": "Upstage, Solar Open Technical Report (arXiv 2601.07022v1, CC BY 4.0), Table 6 'English Benchmarks', row 'Tau 2 Telecom', column 'Solar Open (102B)' = 55.6. Upstage's own model; same table as the existing gpt-oss-120b (high) 57.7 and GLM-4.5-Air 28.1 rows of this run. Checkpoint upstage/Solar-Open-100B (the card cites arXiv 2601.07022).",
    "source_id": "report-2601.07022v1"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "mmlu-pro",
    "score": 0.838,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'MMLU-Pro', column 'K-EXAONE (Reasoning)' = 83.8. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "gpqa-diamond",
    "score": 0.791,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'GPQA-Diamond', column 'K-EXAONE (Reasoning)' = 79.1. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "hle",
    "score": 0.136,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'Humanity's Last Exam', column 'K-EXAONE (Reasoning)' = 13.6. The report says HLE is 'the text-only subset'. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "hle",
    "score": 0.106,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'Humanity's Last Exam', column 'EXAONE 4.0 (Reasoning)' = 10.6. The report says HLE is 'the text-only subset'. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "imo-answerbench",
    "score": 0.763,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'IMO-AnswerBench', column 'K-EXAONE (Reasoning)' = 76.3. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "imo-answerbench",
    "score": 0.661,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'IMO-AnswerBench', column 'EXAONE 4.0 (Reasoning)' = 66.1. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "aime-2025",
    "score": 0.928,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'AIME 2025', column 'K-EXAONE (Reasoning)' = 92.8. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "hmmt-nov-2025",
    "score": 0.868,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'HMMT Nov 2025', column 'K-EXAONE (Reasoning)' = 86.8. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "hmmt-nov-2025",
    "score": 0.781,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'HMMT Nov 2025', column 'EXAONE 4.0 (Reasoning)' = 78.1. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.259,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'LiveCodeBench Pro 25Q2 (Medium)', column 'K-EXAONE (Reasoning)' = 25.9. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.048,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'LiveCodeBench Pro 25Q2 (Medium)', column 'EXAONE 4.0 (Reasoning)' = 4.8. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.807,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'LiveCodeBench v6', column 'K-EXAONE (Reasoning)' = 80.7. LiveCodeBench v6 date window not stated. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.667,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'LiveCodeBench v6', column 'EXAONE 4.0 (Reasoning)' = 66.7. LiveCodeBench v6 date window not stated. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "tau2-retail",
    "score": 0.786,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'τ2-Bench (Retail)', column 'K-EXAONE (Reasoning)' = 78.6. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-retail",
    "score": 0.675,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'τ2-Bench (Retail)', column 'EXAONE 4.0 (Reasoning)' = 67.5. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "tau2-airline",
    "score": 0.604,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'τ2-Bench (Airline)', column 'K-EXAONE (Reasoning)' = 60.4. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-airline",
    "score": 0.52,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'τ2-Bench (Airline)', column 'EXAONE 4.0 (Reasoning)' = 52.0. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "tau2-telecom",
    "score": 0.735,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'τ2-Bench (Telecom)', column 'K-EXAONE (Reasoning)' = 73.5. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "ifbench-unspecified",
    "score": 0.673,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'IFBench', column 'K-EXAONE (Reasoning)' = 67.3. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "ifbench-unspecified",
    "score": 0.36,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'IFBench', column 'EXAONE 4.0 (Reasoning)' = 36.0. Column 'EXAONE 4.0 (Reasoning)', Architecture 'Dense', Total Params '32B' = exaone-4-0-32b (same mapping as the existing τ2-Telecom row from this card). Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "ifeval-unspecified",
    "score": 0.897,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B/blob/61e6d578eb102b578e5704e2916ac841df9eca0a/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-08-03",
    "note": "LG's own figure (rule b): K-EXAONE-236B-A23B card (@61e6d57), 'Evaluation Results' table ('evaluation results of the K-EXAONE model in reasoning mode, compared to our previous model, EXAONE-4.0'), row 'IFEval', column 'K-EXAONE (Reasoning)' = 89.7. Details in the K-EXAONE technical report (arXiv 2601.01739, CC BY-NC-ND; read for setup only: temperature 1.0, top-p 0.95). Other labs' columns not used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "mmlu-pro",
    "score": 0.835,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'MMLU-Pro', column 'K-EXAONE 2.0' = 83.5. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "mmlu-pro",
    "score": 0.838,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'MMLU-Pro', column 'K-EXAONE (236B / 23B)' = 83.8. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "glm-5-1",
    "bench": "mmlu-pro",
    "score": 0.86,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'MMLU-Pro', column 'GLM-5.1' = 86.0. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'GLM-5.1' (754B / 40B active) = zai-org/GLM-5.1 (glm-5-1), reasoning mode. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "gpqa-diamond",
    "score": 0.822,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'GPQA-Diamond', column 'K-EXAONE 2.0' = 82.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "gpqa-diamond",
    "score": 0.791,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'GPQA-Diamond', column 'K-EXAONE (236B / 23B)' = 79.1. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "hle",
    "score": 0.183,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'Humanity's Last Exam', column 'K-EXAONE 2.0' = 18.3. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. HLE: 'We use the text-only subset' (report). LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "hle",
    "score": 0.136,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'Humanity's Last Exam', column 'K-EXAONE (236B / 23B)' = 13.6. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. HLE: 'We use the text-only subset' (report). Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "aime-2026",
    "score": 0.923,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'AIME 2026', column 'K-EXAONE 2.0' = 92.3. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "aime-2026",
    "score": 0.922,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'AIME 2026', column 'K-EXAONE (236B / 23B)' = 92.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "aime-2026",
    "score": 0.952,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'AIME 2026', column 'DSV4 Pro (max)' = 95.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'DSV4 Pro (max)' (1.6T / 49B active), report 'DeepSeek V4 Pro (Reasoning: max)'; the card (2026-07-29) and report (2026-08-05) predate DeepSeek-V4-Pro-0813, so this is the April 2026 preview weights (deepseek-v4-pro-preview), max reasoning. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "hmmt-feb-2026",
    "score": 0.784,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'HMMT Feb 2026', column 'K-EXAONE 2.0' = 78.4. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "hmmt-feb-2026",
    "score": 0.807,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'HMMT Feb 2026', column 'K-EXAONE (236B / 23B)' = 80.7. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hmmt-feb-2026",
    "score": 0.846,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'HMMT Feb 2026', column 'Qwen3.5' = 84.6. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'Qwen3.5' (397B / 17B active) = Qwen/Qwen3.5-397B-A17B, reasoning mode. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "imo-answerbench",
    "score": 0.786,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IMO Answer', column 'K-EXAONE 2.0' = 78.6. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "imo-answerbench",
    "score": 0.763,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IMO Answer', column 'K-EXAONE (236B / 23B)' = 76.3. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "swe-bench-verified",
    "score": 0.682,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'SWE Bench Verified', column 'K-EXAONE 2.0' = 68.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "swe-bench-verified",
    "score": 0.494,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'SWE Bench Verified', column 'K-EXAONE (236B / 23B)' = 49.4. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "glm-5-1",
    "bench": "swe-bench-verified",
    "score": 0.736,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'SWE Bench Verified', column 'GLM-5.1' = 73.6. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'GLM-5.1' (754B / 40B active) = zai-org/GLM-5.1 (glm-5-1), reasoning mode. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "terminal-bench-2.1",
    "score": 0.438,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'Terminal-Bench 2.1', column 'K-EXAONE 2.0' = 43.8. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "terminal-bench-2.1",
    "score": 0.303,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'Terminal-Bench 2.1', column 'K-EXAONE (236B / 23B)' = 30.3. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "tau3-banking",
    "score": 0.142,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'τ3-Banking', column 'K-EXAONE 2.0' = 14.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "tau3-banking",
    "score": 0.142,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'τ3-Banking', column 'K-EXAONE (236B / 23B)' = 14.2. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "ifeval-unspecified",
    "score": 0.924,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFEval', column 'K-EXAONE 2.0' = 92.4. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "ifeval-unspecified",
    "score": 0.897,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFEval', column 'K-EXAONE (236B / 23B)' = 89.7. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "glm-5-1",
    "bench": "ifeval-unspecified",
    "score": 0.939,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFEval', column 'GLM-5.1' = 93.9. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'GLM-5.1' (754B / 40B active) = zai-org/GLM-5.1 (glm-5-1), reasoning mode. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "ifeval-unspecified",
    "score": 0.94,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "independent",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFEval', column 'DSV4 Pro (max)' = 94.0. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Column 'DSV4 Pro (max)' (1.6T / 49B active), report 'DeepSeek V4 Pro (Reasoning: max)'; the card (2026-07-29) and report (2026-08-05) predate DeepSeek-V4-Pro-0813, so this is the April 2026 preview weights (deepseek-v4-pro-preview), max reasoning. This cell carries no asterisk in report Table 6, i.e. LG's own internal-environment run (the card itself prints no asterisks; all values in this row equal the report's).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-2-0",
    "bench": "ifbench-unspecified",
    "score": 0.726,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFBench', column 'K-EXAONE 2.0' = 72.6. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. LG's own model, reasoning mode (default enable_thinking=True).",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "ifbench-unspecified",
    "score": 0.673,
    "source": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/blob/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md",
    "by": "lab",
    "date": "2026-07-29",
    "note": "LG AI Research, K-EXAONE-2.0-750B-A37B model card (Apache-2.0, @fec0c8d), 'Evaluation Results' table, row 'IFBench', column 'K-EXAONE (236B / 23B)' = 67.3. Same table as Table 6 of the technical report (arXiv 2608.04505v1), 'The main evaluation results of K-EXAONE 2.0 Reasoning mode'; report: 'For baseline models, when official scores are unavailable, we evaluate them in our internal environment with inference parameters set to the recommended configuration for each model' and 'Asterisk (*) indicates that the scores are from each baseline model's official technical report, blog or leaderboard'. Same-lab previous model = LGAI-EXAONE/K-EXAONE-236B-A23B, reasoning mode.",
    "source_id": "card3-lgai-exaone-k-exaone-2-0-750b-a37b"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.099,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Terminal Bench Hard', column 'HyperCLOVA X 32B Think' = 9.9. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Sec. 5.4: 'TerminalBench Hard, which corresponds to the 47 challenging tasks from the artificial analysis split'; App. C.2: 'all evaluated models utilized the same Terminus-2 agent implementation with reasoning enabled throughout the evaluation'. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "tau2-telecom",
    "score": 0.651,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Telecom', column 'HyperCLOVA X 32B Think' = 65.1. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "tau2-telecom",
    "score": 0.333,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Telecom', column 'Qwen3-VL 32B-Thinking' = 33.3. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Qwen3-VL 32B-Thinking = Qwen/Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking), same mapping as this source's existing TB-Hard row.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "tau2-retail",
    "score": 0.716,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Retail', column 'HyperCLOVA X 32B Think' = 71.6. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-retail",
    "score": 0.595,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Retail', column 'EXAONE 4.0 32B' = 59.5. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. EXAONE 4.0 32B (LG) = exaone-4-0-32b, same mapping as this source's existing rows.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "tau2-retail",
    "score": 0.57,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Retail', column 'Qwen3-VL 32B-Thinking' = 57.0. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Qwen3-VL 32B-Thinking = Qwen/Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking), same mapping as this source's existing TB-Hard row.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "tau2-airline",
    "score": 0.58,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Airline', column 'HyperCLOVA X 32B Think' = 58.0. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "tau2-airline",
    "score": 0.456,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Airline', column 'EXAONE 4.0 32B' = 45.6. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. EXAONE 4.0 32B (LG) = exaone-4-0-32b, same mapping as this source's existing rows.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "tau2-airline",
    "score": 0.544,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Agent / English), row 'Tau 2 – Airline', column 'Qwen3-VL 32B-Thinking' = 54.4. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Qwen3-VL 32B-Thinking = Qwen/Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking), same mapping as this source's existing TB-Hard row.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "gpt-5-1",
    "bench": "tau2-retail",
    "score": 0.781,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Retail', column 'GPT-5.1 (Medium)' = 78.1. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Reasoning effort medium (the only setting printed); same mapping as the existing τ2-Telecom row.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-retail",
    "score": 0.514,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Retail', column 'Qwen3 235B-A22B' = 51.4. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Hybrid April-2025 Qwen3-235B-A22B in reasoning mode, same mapping as the existing τ2-Telecom row (Table 6 HCX values equal Table 5's).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "gpt-5-1",
    "bench": "tau2-airline",
    "score": 0.64,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Airline', column 'GPT-5.1 (Medium)' = 64. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Reasoning effort medium (the only setting printed); same mapping as the existing τ2-Telecom row.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-airline",
    "score": 0.368,
    "source": "https://arxiv.org/html/2601.03286v1",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 6, block 'User-simulator: GPT-4.1', row 'Tau 2 – Airline', column 'Qwen3 235B-A22B' = 36.8. Sec. 5: 'we test all models in this report using Omni-Evaluator'. User simulator GPT-4.1 (Table 5 caption; Table 6 block 'User-simulator: GPT-4.1'); App. C.1/Table 8: max turns 200, temperature 0.6, 'All models were evaluated in reasoning mode'. Hybrid April-2025 Qwen3-235B-A22B in reasoning mode, same mapping as the existing τ2-Telecom row (Table 6 HCX values equal Table 5's).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "mmlu",
    "score": 0.877,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'MMLU', column 'HyperCLOVA X 32B Think' = 87.7. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "mmlu",
    "score": 0.899,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'MMLU', column 'EXAONE 4.0 32B' = 89.9. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. EXAONE 4.0 32B = exaone-4-0-32b ('we also include EXAONE 4.0 32B as a text-only baseline').",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmlu",
    "score": 0.715,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'MMLU', column 'Qwen3-VL 32B-Thinking' = 71.5. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "hellaswag",
    "score": 0.572,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'HellaSwag', column 'HyperCLOVA X 32B Think' = 57.2. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "hellaswag",
    "score": 0.657,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'HellaSwag', column 'EXAONE 4.0 32B' = 65.7. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. EXAONE 4.0 32B = exaone-4-0-32b ('we also include EXAONE 4.0 32B as a text-only baseline').",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "hellaswag",
    "score": 0.621,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'HellaSwag', column 'Qwen3-VL 32B-Thinking' = 62.1. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "piqa",
    "score": 0.767,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "lab",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'PIQA', column 'HyperCLOVA X 32B Think' = 76.7. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. NAVER's own model (by lab); checkpoint naver-hyperclovax/HyperCLOVAX-SEED-Think-32B.",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "exaone-4-0-32b",
    "bench": "piqa",
    "score": 0.757,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'PIQA', column 'EXAONE 4.0 32B' = 75.7. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. EXAONE 4.0 32B = exaone-4-0-32b ('we also include EXAONE 4.0 32B as a text-only baseline').",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "piqa",
    "score": 0.88,
    "source": "https://arxiv.org/html/2601.03286v1#S5.T5",
    "by": "independent",
    "date": "2026-01-03",
    "note": "NAVER Cloud HyperCLOVA X team, 'HyperCLOVA X 32B Think' (arXiv 2601.03286v1, CC BY 4.0), Table 5 (Text-to-Text / English), row 'PIQA', column 'Qwen3-VL 32B-Thinking' = 88.0. Sec. 5: 'we test all models in this report using Omni-Evaluator'. Text-to-Text English block; same answer-parsing rule for all models; thinking/reasoning setting of each model not printed per column; shots not printed. Qwen3-VL-32B-Thinking (qwen3-vl-32b-thinking).",
    "source_id": "paper-2601.03286v1-hyperclova-x-32b-think"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.711,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B', column 'LiveCodeBench v6 2408 - 2505' = 71.1. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.657,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B', column 'LiveCodeBench Pro 25Q2 Easy' = 65.7. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.064,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B', column 'LiveCodeBench Pro 25Q2 Med' = 6.4. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b-thinking",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.714,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B-Thinking', column 'LiveCodeBench v6 2408 - 2505' = 71.4. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b-thinking",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.648,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B-Thinking', column 'LiveCodeBench Pro 25Q2 Easy' = 64.8. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b-thinking",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.061,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-8B-Thinking', column 'LiveCodeBench Pro 25Q2 Med' = 6.1. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.746,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-14B-Thinking', column 'LiveCodeBench v6 2408 - 2505' = 74.6. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.689,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-14B-Thinking', column 'LiveCodeBench Pro 25Q2 Easy' = 68.9. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.105,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Nemotron-Cascade-14B-Thinking', column 'LiveCodeBench Pro 25Q2 Med' = 10.5. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "openreasoning-nemotron-32b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.702,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'OpenReasoning-Nemotron-32B', column 'LiveCodeBench v6 2408 - 2505' = 70.2. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "openreasoning-nemotron-32b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.662,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'OpenReasoning-Nemotron-32B', column 'LiveCodeBench Pro 25Q2 Easy' = 66.2. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "openreasoning-nemotron-32b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.083,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'OpenReasoning-Nemotron-32B', column 'LiveCodeBench Pro 25Q2 Med' = 8.3. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-14b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.587,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.0-14B', column 'LiveCodeBench v6 2408 - 2505' = 58.7. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-14b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.479,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.0-14B', column 'LiveCodeBench Pro 25Q2 Easy' = 47.9. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-14b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.022,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.0-14B', column 'LiveCodeBench Pro 25Q2 Med' = 2.2. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-1-1-7b",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.552,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.1-7B', column 'LiveCodeBench v6 2408 - 2505' = 55.2. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-1-1-7b",
    "bench": "livecodebench-pro-2025q2-easy",
    "score": 0.423,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.1-7B', column 'LiveCodeBench Pro 25Q2 Easy' = 42.3. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Easy, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "acereason-nemotron-1-1-7b",
    "bench": "livecodebench-pro-2025q2-medium",
    "score": 0.018,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'AceReason-Nemotron-1.1-7B', column 'LiveCodeBench Pro 25Q2 Med' = 1.8. LiveCodeBench Pro 2025Q2 split (2025/04-2025/07), Medium, pass@1 (avg@8), thinking mode. NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.653,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'NVIDIA-Nemotron-Nano-9B-v2', column 'LiveCodeBench v6 2408 - 2505' = 65.3. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "llama-3-3-nemotron-super-49b-v1-5",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.681,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 9 'Competitive programming results on comprehensive benchmarks, evaluated against a significantly expanded set of proprietary and open-source baseline models', row 'Llama-3.3-Nemotron-Super-49B-v1.5', column 'LiveCodeBench v6 2408 - 2505' = 68.1. Column header 'LiveCodeBench / v6 / 2408 - 2505' (pass@1 avg@k, thinking mode). NVIDIA's own model (by lab). Appendix B: 'For our Nemotron-Cascade models evaluated on LiveCodeBench (v5/v6) and LiveCodeBench Pro, we use a 64K-token thinking budget, a sampling temperature of 0.6, a top-p of 0.95'; 'We evaluate baseline models with their recommended inference configurations, ensuring a thinking budget of at least 64K tokens.'",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "mmlu",
    "score": 0.837,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU', column 'Nemotron Cascade-8B' = 83.7. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "mmlu",
    "score": 0.851,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU', column 'Nemotron-Cascade 14B-Thinking' = 85.1. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "mmlu",
    "score": 0.826,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU', column 'Nemotron-Nano 9B-v2' = 82.6. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "mmlu-pro",
    "score": 0.757,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU-Pro', column 'Nemotron Cascade-8B' = 75.7. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "mmlu-pro",
    "score": 0.77,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU-Pro', column 'Nemotron-Cascade 14B-Thinking' = 77.0. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "mmlu-pro",
    "score": 0.733,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'MMLU-Pro', column 'Nemotron-Nano 9B-v2' = 73.3. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "gpqa-diamond",
    "score": 0.665,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'GPQA-Diamond', column 'Nemotron Cascade-8B' = 66.5. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.696,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'GPQA-Diamond', column 'Nemotron-Cascade 14B-Thinking' = 69.6. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "gpqa-diamond",
    "score": 0.64,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'GPQA-Diamond', column 'Nemotron-Nano 9B-v2' = 64.0. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "ifeval-prompt-strict",
    "score": 0.902,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFEval (strict prompt)', column 'Nemotron Cascade-8B' = 90.2. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "ifeval-prompt-strict",
    "score": 0.819,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFEval (strict prompt)', column 'Nemotron-Cascade 14B-Thinking' = 81.9. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "ifeval-prompt-strict",
    "score": 0.861,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFEval (strict prompt)', column 'Nemotron-Nano 9B-v2' = 86.1. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "ifbench-unspecified",
    "score": 0.408,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFBench', column 'Nemotron Cascade-8B' = 40.8. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption; aggregation not stated. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "ifbench-unspecified",
    "score": 0.417,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFBench', column 'Nemotron-Cascade 14B-Thinking' = 41.7. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption; aggregation not stated. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "ifbench-unspecified",
    "score": 0.374,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'IFBench', column 'Nemotron-Nano 9B-v2' = 37.4. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Higher of thinking/non-thinking mode per caption; aggregation not stated. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "aime-2024",
    "score": 0.895,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2024 (no tools)', column 'Nemotron Cascade-8B' = 89.5. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "aime-2024",
    "score": 0.897,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2024 (no tools)', column 'Nemotron-Cascade 14B-Thinking' = 89.7. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2024",
    "score": 0.819,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2024 (no tools)', column 'Nemotron-Nano 9B-v2' = 81.9. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "aime-2025",
    "score": 0.801,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2025 (no tools)', column 'Nemotron Cascade-8B' = 80.1. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "aime-2025",
    "score": 0.833,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2025 (no tools)', column 'Nemotron-Cascade 14B-Thinking' = 83.3. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "aime-2025",
    "score": 0.72,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'AIME 2025 (no tools)', column 'Nemotron-Nano 9B-v2' = 72.0. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. avg@k pass@1. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "swe-bench-verified",
    "score": 0.372,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'SWE Verified (Agentless)', column 'Nemotron Cascade-8B' = 37.2. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Agentless scaffold, 32K thinking budget (App. B); not the test-time-scaling row. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "swe-bench-verified",
    "score": 0.431,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'SWE Verified (Agentless)', column 'Nemotron-Cascade 14B-Thinking' = 43.1. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Agentless scaffold, 32K thinking budget (App. B); not the test-time-scaling row. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-8b",
    "bench": "bfcl-v3",
    "score": 0.644,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'BFCL V3', column 'Nemotron Cascade-8B' = 64.4. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-cascade-14b-thinking",
    "bench": "bfcl-v3",
    "score": 0.675,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'BFCL V3', column 'Nemotron-Cascade 14B-Thinking' = 67.5. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "bfcl-v3",
    "score": 0.669,
    "source": "https://arxiv.org/html/2512.13607v2",
    "by": "lab",
    "date": "2026-03-27",
    "note": "NVIDIA Nemotron-Cascade report (arXiv 2512.13607v2, CC BY 4.0), Table 1 'Main results', row 'BFCL V3', column 'Nemotron-Nano 9B-v2' = 66.9. Caption: 'For unified reasoning models, we report reasoning-related benchmark results in thinking mode. For IFEval and IFBench, we report the higher score obtained from either thinking or non-thinking mode.' All scores pass@1 averaged over k generations; Nemotron-Cascade models: 64K max generation, temperature 0.6, top-p 0.95. Nano-9B-v2 is NVIDIA's earlier model; the report states 'For baseline models, we use officially reported results whenever available', so this may be NVIDIA's own published figure rather than a re-run (still lab). NVIDIA's own model (by lab).",
    "source_id": "report-2512.13607v2"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmmu",
    "score": 0.787,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'MMMU', column 'Qwen3-VL 235B-A22B / instruct' = 78.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmmu-pro",
    "score": 0.681,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'MMMU-Pro', column 'Qwen3-VL 235B-A22B / instruct' = 68.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mathvista-mini",
    "score": 0.849,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'MathVista mini', column 'Qwen3-VL 235B-A22B / instruct' = 84.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "realworldqa",
    "score": 0.792,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'RealWorldQA', column 'Qwen3-VL 235B-A22B / instruct' = 79.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mmstar",
    "score": 0.784,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'MMStar', column 'Qwen3-VL 235B-A22B / instruct' = 78.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "hallusionbench",
    "score": 0.632,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'HallusionBench', column 'Qwen3-VL 235B-A22B / instruct' = 63.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "docvqa",
    "score": 0.965,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'DocVQA test', column 'Qwen3-VL 235B-A22B / thinking' = 96.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "docvqa",
    "score": 0.971,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'DocVQA test', column 'Qwen3-VL 235B-A22B / instruct' = 97.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "chartqa",
    "score": 0.903,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'ChartQA test', column 'Qwen3-VL 235B-A22B / thinking' = 90.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "chartqa",
    "score": 0.903,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'ChartQA test', column 'Qwen3-VL 235B-A22B / instruct' = 90.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "ocrbench",
    "score": 0.92,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'OCRBench', column 'Qwen3-VL 235B-A22B / instruct' = 920. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "cc-ocr",
    "score": 0.822,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'CC-OCR', column 'Qwen3-VL 235B-A22B / instruct' = 82.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "mvbench",
    "score": 0.765,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'MVBench', column 'Qwen3-VL 235B-A22B / instruct' = 76.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.792,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'Video-MME w/o sub.', column 'Qwen3-VL 235B-A22B / instruct' = 79.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "lvbench",
    "score": 0.677,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'LVBench', column 'Qwen3-VL 235B-A22B / instruct' = 67.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "screenspot-pro",
    "score": 0.618,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'ScreenSpot Pro', column 'Qwen3-VL 235B-A22B / thinking' = 61.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "screenspot-pro",
    "score": 0.62,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'ScreenSpot Pro', column 'Qwen3-VL 235B-A22B / instruct' = 62. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "androidworld",
    "score": 0.62,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'AndroidWorld', column 'Qwen3-VL 235B-A22B / thinking' = 62. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "androidworld",
    "score": 0.637,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=15",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 2 'Performance of Qwen3-VL-235B-A22B and top-tier models on visual benchmarks' on PDF page 15, row 'AndroidWorld', column 'Qwen3-VL 235B-A22B / instruct' = 63.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mmmu",
    "score": 0.76,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU', column 'Qwen3-VL 30B-A3B / thinking' = 76. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mmmu",
    "score": 0.742,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU', column 'Qwen3-VL 30B-A3B / instruct' = 74.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmmu",
    "score": 0.781,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU', column 'Qwen3-VL 32B / thinking' = 78.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mmmu",
    "score": 0.76,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU', column 'Qwen3-VL 32B / instruct' = 76. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mmmu-pro",
    "score": 0.63,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU-Pro', column 'Qwen3-VL 30B-A3B / thinking' = 63. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mmmu-pro",
    "score": 0.604,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU-Pro', column 'Qwen3-VL 30B-A3B / instruct' = 60.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmmu-pro",
    "score": 0.681,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU-Pro', column 'Qwen3-VL 32B / thinking' = 68.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mmmu-pro",
    "score": 0.653,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMMU-Pro', column 'Qwen3-VL 32B / instruct' = 65.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mathvista-mini",
    "score": 0.819,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MathVista mini', column 'Qwen3-VL 30B-A3B / thinking' = 81.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mathvista-mini",
    "score": 0.801,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MathVista mini', column 'Qwen3-VL 30B-A3B / instruct' = 80.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mathvista-mini",
    "score": 0.859,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MathVista mini', column 'Qwen3-VL 32B / thinking' = 85.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mathvista-mini",
    "score": 0.838,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MathVista mini', column 'Qwen3-VL 32B / instruct' = 83.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "realworldqa",
    "score": 0.774,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'RealWorldQA', column 'Qwen3-VL 30B-A3B / thinking' = 77.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "realworldqa",
    "score": 0.737,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'RealWorldQA', column 'Qwen3-VL 30B-A3B / instruct' = 73.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "realworldqa",
    "score": 0.784,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'RealWorldQA', column 'Qwen3-VL 32B / thinking' = 78.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "realworldqa",
    "score": 0.79,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'RealWorldQA', column 'Qwen3-VL 32B / instruct' = 79. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mmstar",
    "score": 0.755,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMStar', column 'Qwen3-VL 30B-A3B / thinking' = 75.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mmstar",
    "score": 0.721,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMStar', column 'Qwen3-VL 30B-A3B / instruct' = 72.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmstar",
    "score": 0.794,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMStar', column 'Qwen3-VL 32B / thinking' = 79.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mmstar",
    "score": 0.777,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MMStar', column 'Qwen3-VL 32B / instruct' = 77.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "hallusionbench",
    "score": 0.66,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'HallusionBench', column 'Qwen3-VL 30B-A3B / thinking' = 66. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "hallusionbench",
    "score": 0.615,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'HallusionBench', column 'Qwen3-VL 30B-A3B / instruct' = 61.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "hallusionbench",
    "score": 0.674,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'HallusionBench', column 'Qwen3-VL 32B / thinking' = 67.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "hallusionbench",
    "score": 0.638,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'HallusionBench', column 'Qwen3-VL 32B / instruct' = 63.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "docvqa",
    "score": 0.955,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'DocVQA test', column 'Qwen3-VL 30B-A3B / thinking' = 95.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "docvqa",
    "score": 0.95,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'DocVQA test', column 'Qwen3-VL 30B-A3B / instruct' = 95. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "docvqa",
    "score": 0.961,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'DocVQA test', column 'Qwen3-VL 32B / thinking' = 96.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "docvqa",
    "score": 0.969,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'DocVQA test', column 'Qwen3-VL 32B / instruct' = 96.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "chartqa",
    "score": 0.894,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ChartQA test', column 'Qwen3-VL 30B-A3B / thinking' = 89.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "chartqa",
    "score": 0.868,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ChartQA test', column 'Qwen3-VL 30B-A3B / instruct' = 86.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "chartqa",
    "score": 0.89,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ChartQA test', column 'Qwen3-VL 32B / thinking' = 89. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "chartqa",
    "score": 0.885,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ChartQA test', column 'Qwen3-VL 32B / instruct' = 88.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "ocrbench",
    "score": 0.839,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'OCRBench', column 'Qwen3-VL 30B-A3B / thinking' = 839. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "ocrbench",
    "score": 0.903,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'OCRBench', column 'Qwen3-VL 30B-A3B / instruct' = 903. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "ocrbench",
    "score": 0.855,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'OCRBench', column 'Qwen3-VL 32B / thinking' = 855. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "ocrbench",
    "score": 0.895,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'OCRBench', column 'Qwen3-VL 32B / instruct' = 895. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "cc-ocr",
    "score": 0.778,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CC-OCR', column 'Qwen3-VL 30B-A3B / thinking' = 77.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "cc-ocr",
    "score": 0.807,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CC-OCR', column 'Qwen3-VL 30B-A3B / instruct' = 80.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "cc-ocr",
    "score": 0.796,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CC-OCR', column 'Qwen3-VL 32B / thinking' = 79.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "cc-ocr",
    "score": 0.803,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CC-OCR', column 'Qwen3-VL 32B / instruct' = 80.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "charxiv-dq",
    "score": 0.869,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(DQ)', column 'Qwen3-VL 30B-A3B / thinking' = 86.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "charxiv-dq",
    "score": 0.855,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(DQ)', column 'Qwen3-VL 30B-A3B / instruct' = 85.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "charxiv-dq",
    "score": 0.902,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(DQ)', column 'Qwen3-VL 32B / thinking' = 90.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "charxiv-dq",
    "score": 0.905,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(DQ)', column 'Qwen3-VL 32B / instruct' = 90.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "charxiv-rq",
    "score": 0.566,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(RQ)', column 'Qwen3-VL 30B-A3B / thinking' = 56.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "charxiv-rq",
    "score": 0.489,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(RQ)', column 'Qwen3-VL 30B-A3B / instruct' = 48.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "charxiv-rq",
    "score": 0.652,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(RQ)', column 'Qwen3-VL 32B / thinking' = 65.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "charxiv-rq",
    "score": 0.628,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'CharXiv(RQ)', column 'Qwen3-VL 32B / instruct' = 62.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mvbench",
    "score": 0.72,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MVBench', column 'Qwen3-VL 30B-A3B / thinking' = 72. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mvbench",
    "score": 0.723,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MVBench', column 'Qwen3-VL 30B-A3B / instruct' = 72.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mvbench",
    "score": 0.732,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MVBench', column 'Qwen3-VL 32B / thinking' = 73.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mvbench",
    "score": 0.728,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'MVBench', column 'Qwen3-VL 32B / instruct' = 72.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.733,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'Video-MME w/o sub.', column 'Qwen3-VL 30B-A3B / thinking' = 73.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.745,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'Video-MME w/o sub.', column 'Qwen3-VL 30B-A3B / instruct' = 74.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.773,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'Video-MME w/o sub.', column 'Qwen3-VL 32B / thinking' = 77.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.766,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'Video-MME w/o sub.', column 'Qwen3-VL 32B / instruct' = 76.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "lvbench",
    "score": 0.592,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'LVBench', column 'Qwen3-VL 30B-A3B / thinking' = 59.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "lvbench",
    "score": 0.625,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'LVBench', column 'Qwen3-VL 30B-A3B / instruct' = 62.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "lvbench",
    "score": 0.626,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'LVBench', column 'Qwen3-VL 32B / thinking' = 62.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "lvbench",
    "score": 0.638,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'LVBench', column 'Qwen3-VL 32B / instruct' = 63.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "screenspot-pro",
    "score": 0.573,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ScreenSpot Pro', column 'Qwen3-VL 30B-A3B / thinking' = 57.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "screenspot-pro",
    "score": 0.605,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ScreenSpot Pro', column 'Qwen3-VL 30B-A3B / instruct' = 60.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "screenspot-pro",
    "score": 0.571,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ScreenSpot Pro', column 'Qwen3-VL 32B / thinking' = 57.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "screenspot-pro",
    "score": 0.579,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'ScreenSpot Pro', column 'Qwen3-VL 32B / instruct' = 57.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "androidworld",
    "score": 0.55,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'AndroidWorld', column 'Qwen3-VL 30B-A3B / thinking' = 55. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "androidworld",
    "score": 0.543,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'AndroidWorld', column 'Qwen3-VL 30B-A3B / instruct' = 54.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "androidworld",
    "score": 0.637,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'AndroidWorld', column 'Qwen3-VL 32B / thinking' = 63.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "androidworld",
    "score": 0.573,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=16",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 3 'Performance of medium-sized Qwen3-VL models and previous models on visual benchmarks' on PDF page 16, row 'AndroidWorld', column 'Qwen3-VL 32B / instruct' = 57.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mmmu",
    "score": 0.614,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 2B / thinking' = 61.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmmu",
    "score": 0.534,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 2B / instruct' = 53.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mmmu",
    "score": 0.708,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 4B / thinking' = 70.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mmmu",
    "score": 0.674,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 4B / instruct' = 67.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mmmu",
    "score": 0.741,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 8B / thinking' = 74.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mmmu",
    "score": 0.696,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU', column 'Qwen3-VL 8B / instruct' = 69.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mmmu-pro",
    "score": 0.425,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 2B / thinking' = 42.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmmu-pro",
    "score": 0.365,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 2B / instruct' = 36.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mmmu-pro",
    "score": 0.57,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 4B / thinking' = 57. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mmmu-pro",
    "score": 0.532,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 4B / instruct' = 53.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mmmu-pro",
    "score": 0.604,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 8B / thinking' = 60.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mmmu-pro",
    "score": 0.559,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMMU-Pro', column 'Qwen3-VL 8B / instruct' = 55.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mathvista-mini",
    "score": 0.736,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 2B / thinking' = 73.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mathvista-mini",
    "score": 0.613,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 2B / instruct' = 61.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mathvista-mini",
    "score": 0.795,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 4B / thinking' = 79.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mathvista-mini",
    "score": 0.737,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 4B / instruct' = 73.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mathvista-mini",
    "score": 0.814,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 8B / thinking' = 81.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mathvista-mini",
    "score": 0.772,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MathVista mini', column 'Qwen3-VL 8B / instruct' = 77.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "realworldqa",
    "score": 0.695,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 2B / thinking' = 69.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "realworldqa",
    "score": 0.639,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 2B / instruct' = 63.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "realworldqa",
    "score": 0.732,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 4B / thinking' = 73.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "realworldqa",
    "score": 0.709,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 4B / instruct' = 70.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "realworldqa",
    "score": 0.735,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 8B / thinking' = 73.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "realworldqa",
    "score": 0.715,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'RealWorldQA', column 'Qwen3-VL 8B / instruct' = 71.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mmstar",
    "score": 0.681,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 2B / thinking' = 68.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmstar",
    "score": 0.583,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 2B / instruct' = 58.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mmstar",
    "score": 0.732,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 4B / thinking' = 73.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mmstar",
    "score": 0.698,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 4B / instruct' = 69.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mmstar",
    "score": 0.753,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 8B / thinking' = 75.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mmstar",
    "score": 0.709,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MMStar', column 'Qwen3-VL 8B / instruct' = 70.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "hallusionbench",
    "score": 0.549,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 2B / thinking' = 54.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "hallusionbench",
    "score": 0.514,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 2B / instruct' = 51.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "hallusionbench",
    "score": 0.641,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 4B / thinking' = 64.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "hallusionbench",
    "score": 0.576,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 4B / instruct' = 57.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "hallusionbench",
    "score": 0.654,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 8B / thinking' = 65.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "hallusionbench",
    "score": 0.611,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'HallusionBench', column 'Qwen3-VL 8B / instruct' = 61.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "docvqa",
    "score": 0.929,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 2B / thinking' = 92.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "docvqa",
    "score": 0.933,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 2B / instruct' = 93.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "docvqa",
    "score": 0.942,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 4B / thinking' = 94.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "docvqa",
    "score": 0.953,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 4B / instruct' = 95.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "docvqa",
    "score": 0.953,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 8B / thinking' = 95.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "docvqa",
    "score": 0.961,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'DocVQA test', column 'Qwen3-VL 8B / instruct' = 96.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "chartqa",
    "score": 0.866,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 2B / thinking' = 86.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "chartqa",
    "score": 0.791,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 2B / instruct' = 79.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "chartqa",
    "score": 0.888,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 4B / thinking' = 88.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "chartqa",
    "score": 0.846,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 4B / instruct' = 84.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "chartqa",
    "score": 0.886,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 8B / thinking' = 88.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "chartqa",
    "score": 0.896,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ChartQA test', column 'Qwen3-VL 8B / instruct' = 89.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "ocrbench",
    "score": 0.792,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 2B / thinking' = 792. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "ocrbench",
    "score": 0.858,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 2B / instruct' = 858. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "ocrbench",
    "score": 0.808,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 4B / thinking' = 808. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "ocrbench",
    "score": 0.881,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 4B / instruct' = 881. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "ocrbench",
    "score": 0.819,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 8B / thinking' = 819. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "ocrbench",
    "score": 0.896,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'OCRBench', column 'Qwen3-VL 8B / instruct' = 896. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). OCRBench printed on its 0-1000 scale; stored /1000. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "cc-ocr",
    "score": 0.683,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 2B / thinking' = 68.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "cc-ocr",
    "score": 0.728,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 2B / instruct' = 72.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "cc-ocr",
    "score": 0.738,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 4B / thinking' = 73.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "cc-ocr",
    "score": 0.762,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 4B / instruct' = 76.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "cc-ocr",
    "score": 0.763,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 8B / thinking' = 76.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "cc-ocr",
    "score": 0.799,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CC-OCR', column 'Qwen3-VL 8B / instruct' = 79.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "charxiv-dq",
    "score": 0.701,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 2B / thinking' = 70.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "charxiv-dq",
    "score": 0.623,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 2B / instruct' = 62.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "charxiv-dq",
    "score": 0.839,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 4B / thinking' = 83.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "charxiv-dq",
    "score": 0.762,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 4B / instruct' = 76.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "charxiv-dq",
    "score": 0.859,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 8B / thinking' = 85.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "charxiv-dq",
    "score": 0.83,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(DQ)', column 'Qwen3-VL 8B / instruct' = 83. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "charxiv-rq",
    "score": 0.371,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 2B / thinking' = 37.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "charxiv-rq",
    "score": 0.268,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 2B / instruct' = 26.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "charxiv-rq",
    "score": 0.503,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 4B / thinking' = 50.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "charxiv-rq",
    "score": 0.397,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 4B / instruct' = 39.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "charxiv-rq",
    "score": 0.53,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 8B / thinking' = 53. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "charxiv-rq",
    "score": 0.464,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'CharXiv(RQ)', column 'Qwen3-VL 8B / instruct' = 46.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mvbench",
    "score": 0.645,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 2B / thinking' = 64.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mvbench",
    "score": 0.617,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 2B / instruct' = 61.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mvbench",
    "score": 0.693,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 4B / thinking' = 69.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mvbench",
    "score": 0.689,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 4B / instruct' = 68.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mvbench",
    "score": 0.69,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 8B / thinking' = 69. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mvbench",
    "score": 0.687,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'MVBench', column 'Qwen3-VL 8B / instruct' = 68.7. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.621,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 2B / thinking' = 62.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.619,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 2B / instruct' = 61.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.689,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 4B / thinking' = 68.9. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.693,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 4B / instruct' = 69.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.718,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 8B / thinking' = 71.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "videomme-without-sub",
    "score": 0.714,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'Video-MME w/o sub.', column 'Qwen3-VL 8B / instruct' = 71.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "lvbench",
    "score": 0.476,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 2B / thinking' = 47.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "lvbench",
    "score": 0.474,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 2B / instruct' = 47.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "lvbench",
    "score": 0.535,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 4B / thinking' = 53.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "lvbench",
    "score": 0.562,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 4B / instruct' = 56.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "lvbench",
    "score": 0.558,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 8B / thinking' = 55.8. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "lvbench",
    "score": 0.58,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'LVBench', column 'Qwen3-VL 8B / instruct' = 58. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "screenspot-pro",
    "score": 0.322,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 2B / thinking' = 32.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "screenspot-pro",
    "score": 0.485,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 2B / instruct' = 48.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "screenspot-pro",
    "score": 0.492,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 4B / thinking' = 49.2. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "screenspot-pro",
    "score": 0.595,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 4B / instruct' = 59.5. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "screenspot-pro",
    "score": 0.466,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 8B / thinking' = 46.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "screenspot-pro",
    "score": 0.546,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'ScreenSpot Pro', column 'Qwen3-VL 8B / instruct' = 54.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "androidworld",
    "score": 0.461,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 2B / thinking' = 46.1. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "androidworld",
    "score": 0.364,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 2B / instruct' = 36.4. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "androidworld",
    "score": 0.52,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 4B / thinking' = 52. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "androidworld",
    "score": 0.453,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 4B / instruct' = 45.3. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "androidworld",
    "score": 0.5,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 8B / thinking' = 50. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "androidworld",
    "score": 0.476,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=18",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 4 'Performance of small-sized Qwen3-VL models and GPT-5-nano on visual benchmarks' on PDF page 18, row 'AndroidWorld', column 'Qwen3-VL 8B / instruct' = 47.6. Split/version as printed only (MMMU: split not stated, the conventional val split; DocVQA/ChartQA: 'test'; Video-MME 'w/o sub.'). Table caption: 'Results marked with an * are sourced from the technical report. + denotes results with tool use.' (no * or + on this Qwen cell). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.743,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=21",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 5 'Comparison among Qwen3-VL-235B-A22B (Instruct) and other baselines' on PDF page 21, row 'GPQA', column 'Qwen3-VL 235B-A22B Instruct' = 74.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.771,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 6 'Comparison among Qwen3-VL-235B-A22B (Thinking) and other reasoning baselines' on PDF page 22, row 'GPQA', column 'Qwen3-VL 235B-A22B Thinking' = 77.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "bfcl-v3",
    "score": 0.718,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 6 'Comparison among Qwen3-VL-235B-A22B (Thinking) and other reasoning baselines' on PDF page 22, row 'BFCL-v3', column 'Qwen3-VL 235B-A22B Thinking' = 71.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "tau2-telecom",
    "score": 0.447,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 6 'Comparison among Qwen3-VL-235B-A22B (Thinking) and other reasoning baselines' on PDF page 22, row 'TAU2-Telecom', column 'Qwen3-VL 235B-A22B Thinking' = 44.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Only the per-domain TAU2-Telecom row is used (no aggregate printed). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mmlu-pro",
    "score": 0.786,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'MMLU-Pro', column 'Qwen3-VL 32B Instruct' = 78.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mmlu-pro",
    "score": 0.778,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'MMLU-Pro', column 'Qwen3-VL 30B-A3B Instruct' = 77.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "mmlu-redux",
    "score": 0.898,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'MMLU-Redux', column 'Qwen3-VL 32B Instruct' = 89.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "mmlu-redux",
    "score": 0.884,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'MMLU-Redux', column 'Qwen3-VL 30B-A3B Instruct' = 88.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.689,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'GPQA', column 'Qwen3-VL 32B Instruct' = 68.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.704,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'GPQA', column 'Qwen3-VL 30B-A3B Instruct' = 70.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "supergpqa",
    "score": 0.546,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'SuperGPQA', column 'Qwen3-VL 32B Instruct' = 54.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "supergpqa",
    "score": 0.531,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'SuperGPQA', column 'Qwen3-VL 30B-A3B Instruct' = 53.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "aime-2025",
    "score": 0.662,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'AIME-25', column 'Qwen3-VL 32B Instruct' = 66.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "aime-2025",
    "score": 0.693,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'AIME-25', column 'Qwen3-VL 30B-A3B Instruct' = 69.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.438,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'LiveCodeBench v6', column 'Qwen3-VL 32B Instruct' = 43.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.426,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'LiveCodeBench v6', column 'Qwen3-VL 30B-A3B Instruct' = 42.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.847,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'IFEval', column 'Qwen3-VL 32B Instruct' = 84.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.858,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'IFEval', column 'Qwen3-VL 30B-A3B Instruct' = 85.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-instruct",
    "bench": "bfcl-v3",
    "score": 0.702,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'BFCL-v3', column 'Qwen3-VL 32B Instruct' = 70.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-instruct",
    "bench": "bfcl-v3",
    "score": 0.663,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 7 'Comparison among Qwen3-VL-32B-Instruct, Qwen3-VL-30B-A3B-Instruct, and corresponding baselines' on PDF page 22, row 'BFCL-v3', column 'Qwen3-VL 30B-A3B Instruct' = 66.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmlu-pro",
    "score": 0.821,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3-VL 32B Thinking' = 82.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mmlu-pro",
    "score": 0.805,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3-VL 30B-A3B Thinking' = 80.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "mmlu-redux",
    "score": 0.919,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3-VL 32B Thinking' = 91.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "mmlu-redux",
    "score": 0.909,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3-VL 30B-A3B Thinking' = 90.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.731,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3-VL 32B Thinking' = 73.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.744,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3-VL 30B-A3B Thinking' = 74.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "supergpqa",
    "score": 0.59,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3-VL 32B Thinking' = 59. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "supergpqa",
    "score": 0.564,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3-VL 30B-A3B Thinking' = 56.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "aime-2025",
    "score": 0.837,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3-VL 32B Thinking' = 83.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "aime-2025",
    "score": 0.831,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3-VL 30B-A3B Thinking' = 83.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.656,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3-VL 32B Thinking' = 65.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.642,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3-VL 30B-A3B Thinking' = 64.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.878,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3-VL 32B Thinking' = 87.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.817,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3-VL 30B-A3B Thinking' = 81.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "bfcl-v3",
    "score": 0.717,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3-VL 32B Thinking' = 71.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "bfcl-v3",
    "score": 0.686,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3-VL 30B-A3B Thinking' = 68.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "tau2-telecom",
    "score": 0.469,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'TAU2-Telecom', column 'Qwen3-VL 32B Thinking' = 46.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Only the per-domain TAU2-Telecom row is used (no aggregate printed). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-telecom",
    "score": 0.263,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'TAU2-Telecom', column 'Qwen3 32B Thinking' = 26.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Only the per-domain TAU2-Telecom row is used (no aggregate printed). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-30b-a3b-thinking",
    "bench": "tau2-telecom",
    "score": 0.272,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'TAU2-Telecom', column 'Qwen3-VL 30B-A3B Thinking' = 27.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Only the per-domain TAU2-Telecom row is used (no aggregate printed). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmlu-pro",
    "score": 0.49,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3-VL 2B Instruct' = 49. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mmlu-pro",
    "score": 0.671,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3-VL 4B Instruct' = 67.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mmlu-pro",
    "score": 0.716,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3-VL 8B Instruct' = 71.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-pro",
    "score": 0.696,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Pro', column 'Qwen3 4B Instruct-2507' = 69.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (mmlu-pro), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmlu-redux",
    "score": 0.665,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3-VL 2B Instruct' = 66.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "mmlu-redux",
    "score": 0.815,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3-VL 4B Instruct' = 81.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "mmlu-redux",
    "score": 0.849,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3-VL 8B Instruct' = 84.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.842,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'MMLU-Redux', column 'Qwen3 4B Instruct-2507' = 84.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (mmlu-redux), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.42,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3-VL 2B Instruct' = 42. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.559,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3-VL 4B Instruct' = 55.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "gpqa-unspecified",
    "score": 0.619,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3-VL 8B Instruct' = 61.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "gpqa-unspecified",
    "score": 0.62,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'GPQA', column 'Qwen3 4B Instruct-2507' = 62. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (gpqa-unspecified), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "supergpqa",
    "score": 0.243,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3-VL 2B Instruct' = 24.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "supergpqa",
    "score": 0.403,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3-VL 4B Instruct' = 40.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "supergpqa",
    "score": 0.445,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3-VL 8B Instruct' = 44.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.428,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'SuperGPQA', column 'Qwen3 4B Instruct-2507' = 42.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (supergpqa), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "aime-2025",
    "score": 0.222,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3-VL 2B Instruct' = 22.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "aime-2025",
    "score": 0.466,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3-VL 4B Instruct' = 46.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "aime-2025",
    "score": 0.459,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3-VL 8B Instruct' = 45.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "aime-2025",
    "score": 0.474,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'AIME-25', column 'Qwen3 4B Instruct-2507' = 47.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (aime-2025), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.203,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3-VL 2B Instruct' = 20.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.379,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3-VL 4B Instruct' = 37.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.393,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3-VL 8B Instruct' = 39.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.351,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'LiveCodeBench v6', column 'Qwen3 4B Instruct-2507' = 35.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (livecodebench-v6), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.682,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3-VL 2B Instruct' = 68.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.823,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3-VL 4B Instruct' = 82.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.837,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3-VL 8B Instruct' = 83.7. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.834,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'IFEval', column 'Qwen3 4B Instruct-2507' = 83.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (ifeval-unspecified), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "bfcl-v3",
    "score": 0.554,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3-VL 2B Instruct' = 55.4. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "bfcl-v3",
    "score": 0.633,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3-VL 4B Instruct' = 63.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "bfcl-v3",
    "score": 0.663,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3-VL 8B Instruct' = 66.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.619,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 9 'Comparison among Qwen3-VL-2B (Instruct), Qwen3-VL-4B (Instruct), Qwen3-VL-8B (Instruct) and corresponding baselines' on PDF page 23, row 'BFCL-v3', column 'Qwen3 4B Instruct-2507' = 61.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Instruct release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used. FLAG: same figure as the existing row from card-qwen--qwen3-4b-instruct-2507 (bfcl-v3), whose run holds only this model and is withheld; recording it in the Qwen vendor run lets it count once.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mmlu-pro",
    "score": 0.623,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Pro', column 'Qwen3-VL 2B Thinking' = 62.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mmlu-pro",
    "score": 0.736,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Pro', column 'Qwen3-VL 4B Thinking' = 73.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mmlu-pro",
    "score": 0.773,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Pro', column 'Qwen3-VL 8B Thinking' = 77.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu-pro",
    "score": 0.581,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Pro', column 'Qwen3 1.7B Thinking' = 58.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "mmlu-pro",
    "score": 0.746,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Pro', column 'Qwen3 8B Thinking' = 74.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "mmlu-redux",
    "score": 0.769,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Redux', column 'Qwen3-VL 2B Thinking' = 76.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "mmlu-redux",
    "score": 0.86,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Redux', column 'Qwen3-VL 4B Thinking' = 86. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "mmlu-redux",
    "score": 0.888,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Redux', column 'Qwen3-VL 8B Thinking' = 88.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "mmlu-redux",
    "score": 0.875,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'MMLU-Redux', column 'Qwen3 8B Thinking' = 87.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.495,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'GPQA', column 'Qwen3-VL 2B Thinking' = 49.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.641,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'GPQA', column 'Qwen3-VL 4B Thinking' = 64.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "gpqa-unspecified",
    "score": 0.699,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'GPQA', column 'Qwen3-VL 8B Thinking' = 69.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-unspecified",
    "score": 0.279,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'GPQA', column 'Qwen3 1.7B Thinking' = 27.9. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "gpqa-unspecified",
    "score": 0.62,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'GPQA', column 'Qwen3 8B Thinking' = 62. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). The report labels the row only 'GPQA' (subset not stated) -> gpqa-unspecified. Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "supergpqa",
    "score": 0.346,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'SuperGPQA', column 'Qwen3-VL 2B Thinking' = 34.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "supergpqa",
    "score": 0.468,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'SuperGPQA', column 'Qwen3-VL 4B Thinking' = 46.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "supergpqa",
    "score": 0.512,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'SuperGPQA', column 'Qwen3-VL 8B Thinking' = 51.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "supergpqa",
    "score": 0.476,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'SuperGPQA', column 'Qwen3 8B Thinking' = 47.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "aime-2025",
    "score": 0.39,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'AIME-25', column 'Qwen3-VL 2B Thinking' = 39. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "aime-2025",
    "score": 0.745,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'AIME-25', column 'Qwen3-VL 4B Thinking' = 74.5. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "aime-2025",
    "score": 0.803,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'AIME-25', column 'Qwen3-VL 8B Thinking' = 80.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "aime-2025",
    "score": 0.368,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'AIME-25', column 'Qwen3 1.7B Thinking' = 36.8. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "aime-2025",
    "score": 0.673,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'AIME-25', column 'Qwen3 8B Thinking' = 67.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.293,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'LiveCodeBench v6', column 'Qwen3-VL 2B Thinking' = 29.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.513,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'LiveCodeBench v6', column 'Qwen3-VL 4B Thinking' = 51.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.586,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'LiveCodeBench v6', column 'Qwen3-VL 8B Thinking' = 58.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.313,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'LiveCodeBench v6', column 'Qwen3 1.7B Thinking' = 31.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.51,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'LiveCodeBench v6', column 'Qwen3 8B Thinking' = 51. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). 'LiveCodeBench v6' with no date window printed -> livecodebench-v6-window-unspecified. Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.751,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'IFEval', column 'Qwen3-VL 2B Thinking' = 75.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.826,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'IFEval', column 'Qwen3-VL 4B Thinking' = 82.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.832,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'IFEval', column 'Qwen3-VL 8B Thinking' = 83.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "ifeval-unspecified",
    "score": 0.85,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'IFEval', column 'Qwen3 8B Thinking' = 85. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Aggregation (strict/loose, prompt/instruction) not stated. Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "bfcl-v3",
    "score": 0.572,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'BFCL-v3', column 'Qwen3-VL 2B Thinking' = 57.2. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "bfcl-v3",
    "score": 0.673,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'BFCL-v3', column 'Qwen3-VL 4B Thinking' = 67.3. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "bfcl-v3",
    "score": 0.63,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'BFCL-v3', column 'Qwen3-VL 8B Thinking' = 63. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (separate published checkpoint). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "bfcl-v3",
    "score": 0.566,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'BFCL-v3', column 'Qwen3 1.7B Thinking' = 56.6. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8b",
    "bench": "bfcl-v3",
    "score": 0.681,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=24",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b), Table 10 'Comparison among Qwen3-VL-2B (Thinking), Qwen3-VL-4B (Thinking), Qwen3-VL-8B (Thinking) and corresponding baselines' on PDF page 24, row 'BFCL-v3', column 'Qwen3 8B Thinking' = 68.1. §5.11 evaluation settings: instruct MoE/32B models temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5; small instruct models temperature 1.0, top-p 1.0, top-k 40, presence penalty 2.0; thinking MoE models temperature 0.6, top-p 0.95, top-k 20; dense thinking models temperature 1.0, top-p 0.95, top-k 20, presence penalty 1.5; max output 32,768 tokens except AIME-25 and LiveCodeBench v6 (81,920). Checkpoint: the report's own Thinking release (hybrid Qwen3 checkpoint in thinking mode; highest reasoning setting printed). PDF sha256 ee075d08e67de1148d6437c6c1d481f7894183b8793905a2deb5f62664f49380 (v2 HTML omits Tables 2-4). Vendor-reported (rule b): Qwen's own evaluation of its own model; other labs' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.692,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'MMBench_DEV_EN_V11' (General VQA), cell '0.692'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.731,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'MMBench_DEV_EN_V11' (General VQA), cell '0.731'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.693,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'MMBench_DEV_EN_V11' (General VQA), cell '0.693'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.744,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'MMBench_DEV_EN_V11' (General VQA), cell '0.744'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmbench-en-dev-v1-1",
    "score": 0.76,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'MMBench_DEV_EN_V11' (General VQA), cell '0.760'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmstar",
    "score": 0.531,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'MMStar' (General VQA), cell '0.531'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mmstar",
    "score": 0.495,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'MMStar' (General VQA), cell '0.495'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmstar",
    "score": 0.529,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'MMStar' (General VQA), cell '0.529'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmstar",
    "score": 0.506,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'MMStar' (General VQA), cell '0.506'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmstar",
    "score": 0.614,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'MMStar' (General VQA), cell '0.614'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "realworldqa",
    "score": 0.583,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'RealWorldQA' (General VQA), cell '0.583'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "realworldqa",
    "score": 0.58,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'RealWorldQA' (General VQA), cell '0.580'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "realworldqa",
    "score": 0.507,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'RealWorldQA' (General VQA), cell '0.507'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "realworldqa",
    "score": 0.646,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'RealWorldQA' (General VQA), cell '0.646'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "realworldqa",
    "score": 0.693,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'RealWorldQA' (General VQA), cell '0.693'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "chartqa",
    "score": 0.791,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'ChartQA_Test' (Chart / Document / OCR), cell '0.791'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "chartqa",
    "score": 0.821,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'ChartQA_Test' (Chart / Document / OCR), cell '0.821'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "chartqa",
    "score": 0.422,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'ChartQA_Test' (Chart / Document / OCR), cell '0.422'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "chartqa",
    "score": 0.693,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'ChartQA_Test' (Chart / Document / OCR), cell '0.693'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "chartqa",
    "score": 0.775,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'ChartQA_Test' (Chart / Document / OCR), cell '0.775'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "ocrbench",
    "score": 0.735,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'OCRBench' (Chart / Document / OCR; printed on a 0-1 scale = raw/1000), cell '0.735'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "ocrbench",
    "score": 0.642,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'OCRBench' (Chart / Document / OCR; printed on a 0-1 scale = raw/1000), cell '0.642'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ocrbench",
    "score": 0.719,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'OCRBench' (Chart / Document / OCR; printed on a 0-1 scale = raw/1000), cell '0.719'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "ocrbench",
    "score": 0.751,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'OCRBench' (Chart / Document / OCR; printed on a 0-1 scale = raw/1000), cell '0.751'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ocrbench",
    "score": 0.861,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'OCRBench' (Chart / Document / OCR; printed on a 0-1 scale = raw/1000), cell '0.861'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "ai2d-test",
    "score": 0.741,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'AI2D_TEST' (Chart / Document / OCR), cell '0.741'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "ai2d-test",
    "score": 0.79,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'AI2D_TEST' (Chart / Document / OCR), cell '0.790'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ai2d-test",
    "score": 0.712,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'AI2D_TEST' (Chart / Document / OCR), cell '0.712'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "ai2d-test",
    "score": 0.713,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'AI2D_TEST' (Chart / Document / OCR), cell '0.713'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ai2d-test",
    "score": 0.752,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'AI2D_TEST' (Chart / Document / OCR), cell '0.752'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "charxiv-dq",
    "score": 0.595,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'CharXiv_DQ' (Chart / Document / OCR; VLMEvalKit's only CharXiv descriptive set is the validation split), cell '0.595'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "countbench",
    "score": 0.737,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'CountBench', cell '0.737'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "countbench",
    "score": 0.645,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'CountBench', cell '0.645'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "countbench",
    "score": 0.534,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'CountBench', cell '0.534'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "countbench",
    "score": 0.848,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'CountBench', cell '0.848'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "countbench",
    "score": 0.805,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'CountBench', cell '0.805'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "hallusionbench",
    "score": 0.652,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'HallusionBench', cell '0.652'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "hallusionbench",
    "score": 0.585,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'HallusionBench', cell '0.585'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "hallusionbench",
    "score": 0.598,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'HallusionBench', cell '0.598'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "hallusionbench",
    "score": 0.673,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'HallusionBench', cell '0.673'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hallusionbench",
    "score": 0.655,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'HallusionBench', cell '0.655'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmlu",
    "score": 0.66,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'MMLU_test' (text-only), cell '0.660'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mmlu",
    "score": 0.355,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'MMLU_test' (text-only), cell '0.355'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmlu",
    "score": 0.692,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'MMLU_test' (text-only), cell '0.692'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmlu",
    "score": 0.63,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'MMLU_test' (text-only), cell '0.630'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu",
    "score": 0.543,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'MMLU_test' (text-only), cell '0.543'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "mmlu-pro",
    "score": 0.475,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'MMLU-Pro_test' (text-only), cell '0.475'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "mmlu-pro",
    "score": 0.286,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'MMLU-Pro_test' (text-only), cell '0.286'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mmlu-pro",
    "score": 0.441,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'MMLU-Pro_test' (text-only), cell '0.441'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "mmlu-pro",
    "score": 0.428,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'MMLU-Pro_test' (text-only), cell '0.428'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mmlu-pro",
    "score": 0.298,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'MMLU-Pro_test' (text-only), cell '0.298'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "multi-if-unspecified",
    "score": 0.47,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'Multi-If' (text-only; aggregation not stated), cell '0.470'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "multi-if-unspecified",
    "score": 0.304,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'Multi-If' (text-only; aggregation not stated), cell '0.304'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "multi-if-unspecified",
    "score": 0.687,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'Multi-If' (text-only; aggregation not stated), cell '0.687'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "multi-if-unspecified",
    "score": 0.523,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'Multi-If' (text-only; aggregation not stated), cell '0.523'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "multi-if-unspecified",
    "score": 0.464,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'Multi-If' (text-only; aggregation not stated), cell '0.464'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "ministral-3-3b",
    "bench": "ifeval-unspecified",
    "score": 0.725,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Ministral-3-3B-Instruct', row 'IFEval' (text-only; aggregation not stated), cell '0.725'. Identity: Ministral 3 3B Instruct exists only as mistralai/Ministral-3-3B-Instruct-2512 (catalogue ministral-3-3b; the Reasoning-2512 variant is a separate id and is not the 'Instruct' column). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "phi-3-5-vision-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.543,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Phi-3.5-vision-instruct', row 'IFEval' (text-only; aggregation not stated), cell '0.543'. Identity: exact HF repo name microsoft/Phi-3.5-vision-instruct (catalogue phi-3-5-vision-instruct, 4.15B params). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifeval-unspecified",
    "score": 0.869,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Gemma-4-E2B-it', row 'IFEval' (text-only; aggregation not stated), cell '0.869'. Identity: google/gemma-4-E2B-it = catalogue gemma-4-e2b model_repo (params_b 5.1). Thinking not used per the blog's 'instruction-tuned, non-reasoning' framing and the 1,024-token cap. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.734,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3-VL-2B-Instruct', row 'IFEval' (text-only; aggregation not stated), cell '0.734'. Identity: exact HF repo name Qwen/Qwen3-VL-2B-Instruct (proposed catalogue qwen3-vl-2b-instruct; card size row '2.2B' vs 2.13B in safetensors metadata); the Instruct checkpoint, not Qwen3-VL-2B-Thinking. Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "ifeval-unspecified",
    "score": 0.679,
    "source": "https://huggingface.co/CohereLabs/North-Micro-Vision-Instruct/blob/fa548d9aa62c1220bfd21fc6cfc465f1bd2a1a40/README.md",
    "by": "independent",
    "date": "2026-08-10",
    "note": "Column 'Qwen3.5-2B-Instruct', row 'IFEval' (text-only; aggregation not stated), cell '0.679'. Identity: there is no Qwen/Qwen3.5-2B-Instruct repo; the post-trained hybrid-thinking checkpoint is Qwen/Qwen3.5-2B (catalogue qwen3-5-2b). 'Instruct' read as the non-thinking use, consistent with the blog's 'instruction-tuned, non-reasoning models' and the 1,024-token cap (same mapping as the existing CharXiv_DQ row of this card). Cohere's own run, Benchmark Results table of the pinned North-Micro-Vision-Instruct model card (Apache-2.0). Card text above the table: 'We ran vision-language and text-only evaluations with VLMEvalKit, capping generation at 1,024 tokens'. Cohere's linked HF blog (methodology only, not used for numbers): comparison models run following 'the authors' recommended vision and text inference settings', non-reasoning, 1,024-token cap. Values printed as fractions (no conversion). Rule (a): openly licensed card, authors' own measurement of another lab's model -> by: independent.",
    "source_id": "card2-coherelabs-north-micro-vision-instruct"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmmu",
    "score": 0.806,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MMMU', column 'Qwen3-VL-235B-A22B' = 80.6. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MMMU': thinking 80.6, instruct 78.7); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmmu-pro",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MMMU-Pro', column 'Qwen3-VL-235B-A22B' = 69.3. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MMMU-Pro': thinking 69.3, instruct 68.1); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "math-vision",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MathVision', column 'Qwen3-VL-235B-A22B' = 74.6. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MathVision': thinking 74.6, instruct 66.5); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mathvista-mini",
    "score": 0.858,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'Mathvista(mini)', column 'Qwen3-VL-235B-A22B' = 85.8. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MathVista mini': thinking 85.8, instruct 84.9); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "we-math",
    "score": 0.748,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'We-Math', column 'Qwen3-VL-235B-A22B' = 74.8. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'We-Math': thinking 74.8, instruct 67.5); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "dynamath",
    "score": 0.828,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'DynaMath', column 'Qwen3-VL-235B-A22B' = 82.8. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'DynaMath': thinking 82.8, instruct 79.4); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "realworldqa",
    "score": 0.813,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'RealWorldQA', column 'Qwen3-VL-235B-A22B' = 81.3. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'RealWorldQA': thinking 81.3, instruct 79.2); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmstar",
    "score": 0.787,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MMStar', column 'Qwen3-VL-235B-A22B' = 78.7. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MMStar': thinking 78.7, instruct 78.4); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "hallusionbench",
    "score": 0.667,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'HallusionBench', column 'Qwen3-VL-235B-A22B' = 66.7. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'HallusionBench': thinking 66.7, instruct 63.2); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "simplevqa",
    "score": 0.613,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'SimpleVQA', column 'Qwen3-VL-235B-A22B' = 61.3. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'SimpleVQA': thinking 61.3, instruct 63); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmlongbench-doc",
    "score": 0.562,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MMLongBench-Doc', column 'Qwen3-VL-235B-A22B' = 56.2. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MMLongBenchDoc': thinking 56.2, instruct 57); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "cc-ocr",
    "score": 0.815,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'CC-OCR', column 'Qwen3-VL-235B-A22B' = 81.5. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'CC-OCR': thinking 81.5, instruct 82.2); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "ai2d-test",
    "score": 0.892,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'AI2D_TEST', column 'Qwen3-VL-235B-A22B' = 89.2. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'AI2D w. M.': thinking 89.2, instruct 89.7); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "ocrbench",
    "score": 0.875,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'OCRBench', column 'Qwen3-VL-235B-A22B' = 87.5. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'OCRBench (875 / 920 on 0-1000)': thinking 87.5, instruct 92); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "erqa",
    "score": 0.525,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'ERQA', column 'Qwen3-VL-235B-A22B' = 52.5. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'ERQA': thinking 52.5, instruct 51.3); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "countbench",
    "score": 0.937,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'CountBench', column 'Qwen3-VL-235B-A22B' = 93.7. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'CountBench': thinking 93.7, instruct 93); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "embspatialbench",
    "score": 0.843,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'EmbSpatialBench', column 'Qwen3-VL-235B-A22B' = 84.3. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'EmbSpatialBench': thinking 84.3, instruct 83.1); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "refspatialbench",
    "score": 0.699,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'RefSpatialBench', column 'Qwen3-VL-235B-A22B' = 69.9. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'RefSpatialBench': thinking 69.9, instruct 65.5); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "videomme-without-sub",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'VideoMME(w/o sub.)', column 'Qwen3-VL-235B-A22B' = 79.0. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'Video-MME w/o sub.': thinking 79, instruct 79.2); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "videommmu",
    "score": 0.8,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'VideoMMMU', column 'Qwen3-VL-235B-A22B' = 80.0. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'VideoMMMU': thinking 80, instruct 74.7); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mlvu",
    "score": 0.838,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MLVU (M-Avg)', column 'Qwen3-VL-235B-A22B' = 83.8. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MLVU M-Avg': thinking 83.8, instruct 84.3); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mvbench",
    "score": 0.752,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MVBench', column 'Qwen3-VL-235B-A22B' = 75.2. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MVBench': thinking 75.2, instruct 76.5); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "lvbench",
    "score": 0.636,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'LVBench', column 'Qwen3-VL-235B-A22B' = 63.6. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'LVBench': thinking 63.6, instruct 67.7); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "mmvu",
    "score": 0.711,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'MMVU', column 'Qwen3-VL-235B-A22B' = 71.1. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'MMVU': thinking 71.1, instruct 68.1); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "qwen3-vl-235b-a22b-thinking",
    "bench": "osworld-verified",
    "score": 0.381,
    "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md",
    "by": "lab",
    "date": "2026-02-16",
    "note": "Qwen/Qwen3.5-397B-A17B model card (Apache-2.0, pinned 8472618), vision-language table, row 'OSWorld-Verified', column 'Qwen3-VL-235B-A22B' = 38.1. Checkpoint pinned per cell: the card's column label does not say Instruct or Thinking, but this value equals the Thinking column of Qwen's own Qwen3-VL Technical Report (arXiv 2511.21631v2, Table 2 p.15, row 'OSWorld': thinking 38.1, instruct 31.6); about 25 of the column's cells match the report's thinking column exactly and none of these matches the instruct value, so the column is Qwen's own Qwen3-VL-235B-A22B-Thinking figures. Cells that match the instruct value (ScreenSpot Pro 62.0, AndroidWorld 63.7) or neither (MMBench 89.7, RefCOCO 91.1) are not used. Same-lab sibling column in Qwen's own table (lone-brief step 3); card footnote 'Empty cells (--) indicate scores not yet available or not applicable'; no cell marked as copied. The same figure is NOT also recorded in the first-party-qwen-scores run (counted once).",
    "source_id": "card-qwen--qwen3.5-397b-a17b"
  },
  {
    "model": "jamba2-mini",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.947,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, 'Last updated on September 22, 2026'), row 'ai21labs/jamba-mini-2': factual consistency rate 94.7% (hallucination rate 5.3%), answer rate 99.6%, average summary length 109.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Label is AI21's exact API id.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "jamba-large-1-7",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.903,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, 'Last updated on September 22, 2026'), row 'ai21labs/jamba-large-1.7-2025-07': factual consistency rate 90.3% (hallucination rate 9.7%), answer rate 98.9%, average summary length 124.8 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Label is AI21's exact API id.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "jamba-mini-1-7",
    "bench": "vectara-hhem-factual-consistency",
    "score": 0.853,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Vectara hallucination leaderboard README @af0e320 (Apache-2.0, 'Last updated on September 22, 2026'), row 'ai21labs/jamba-mini-1.7-2025-07': factual consistency rate 85.3% (hallucination rate 14.7%), answer rate 99.1%, average summary length 136.4 words. HHEM-2.3 judge on Vectara's current summarisation set; rates are over answered documents. Label is AI21's exact API id.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.54,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'Fish Audio S2', column SIM = 0.54 (± printed; speaker-level cluster bootstrap, 1,000 resamples). Same authors, toolkit and clean LibriTTS condition as the v2 rows already recorded; the ten v2 systems other than CosyVoice print identical LibriTTS SIM/WER in v3. Identity: fish_audio_s2_ots.yaml (model 'fish_audio_s2', name 'FishAudioS2', HTTP client) + docs/fish_s2_http.md at 42c12ad: the official fish-speech tools.api_server is launched with '--llama-checkpoint-path /absolute/path/to/s2-pro --decoder-checkpoint-path .../s2-pro/codec.pth' (measured source revision 214da3c), i.e. the open fishaudio/s2-pro weights = catalogue fish-audio-s2-pro. The docs call it an extension entry outside the paper's 18-model set; v3 Table 13 caption lists it as an extension system.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.04,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'Fish Audio S2', column WER = 0.04 (fraction as printed, Whisper-medium). Identity: fish_audio_s2_ots.yaml (model 'fish_audio_s2', name 'FishAudioS2', HTTP client) + docs/fish_s2_http.md at 42c12ad: the official fish-speech tools.api_server is launched with '--llama-checkpoint-path /absolute/path/to/s2-pro --decoder-checkpoint-path .../s2-pro/codec.pth' (measured source revision 214da3c), i.e. the open fishaudio/s2-pro weights = catalogue fish-audio-s2-pro. The docs call it an extension entry outside the paper's 18-model set; v3 Table 13 caption lists it as an extension system.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.57,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'MOSS-TTS v1.5', column SIM = 0.57 (± printed; speaker-level cluster bootstrap, 1,000 resamples). Same authors, toolkit and clean LibriTTS condition as the v2 rows already recorded; the ten v2 systems other than CosyVoice print identical LibriTTS SIM/WER in v3. Identity: moss_tts_ots.yaml at 42c12ad: model 'moss_tts', checkpoint 'checkpoints/MOSS-TTS-v1.5' = OpenMOSS-Team/MOSS-TTS-v1.5 (catalogue moss-tts-v1-5); distinct from the paper's 'MOSS-TTSD v0.7' row (moss_ttsd_ots.yaml, not imported).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.06,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'MOSS-TTS v1.5', column WER = 0.06 (fraction as printed, Whisper-medium). Identity: moss_tts_ots.yaml at 42c12ad: model 'moss_tts', checkpoint 'checkpoints/MOSS-TTS-v1.5' = OpenMOSS-Team/MOSS-TTS-v1.5 (catalogue moss-tts-v1-5); distinct from the paper's 'MOSS-TTSD v0.7' row (moss_ttsd_ots.yaml, not imported).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "dots-tts-soar",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.6,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'dots.tts', column SIM = 0.60 (± printed; speaker-level cluster bootstrap, 1,000 resamples). Same authors, toolkit and clean LibriTTS condition as the v2 rows already recorded; the ten v2 systems other than CosyVoice print identical LibriTTS SIM/WER in v3. Identity: dots_tts_ots.yaml at 42c12ad: model 'dots_tts', checkpoint 'checkpoints/dots.tts-soar' = dots-studio/dots.tts-soar (catalogue dots-tts-soar), not the Pretrain or MeanFlow checkpoints.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "dots-tts-soar",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.06,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'dots.tts', column WER = 0.06 (fraction as printed, Whisper-medium). Identity: dots_tts_ots.yaml at 42c12ad: model 'dots_tts', checkpoint 'checkpoints/dots.tts-soar' = dots-studio/dots.tts-soar (catalogue dots-tts-soar), not the Pretrain or MeanFlow checkpoints.",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.56,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'Higgs TTS 3', column SIM = 0.56 (± printed; speaker-level cluster bootstrap, 1,000 resamples). Same authors, toolkit and clean LibriTTS condition as the v2 rows already recorded; the ten v2 systems other than CosyVoice print identical LibriTTS SIM/WER in v3. Identity: higgs_tts3_ots.yaml at 42c12ad: model 'higgs_tts_3', request_model 'bosonai/higgs-tts-3-4b' = catalogue higgs-audio-v3-tts (model_repo bosonai/higgs-tts-3-4b); distinct from the paper's 'Higgs' row (Higgs Audio v2, already recorded).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "higgs-audio-v3-tts",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.05,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'Higgs TTS 3', column WER = 0.05 (fraction as printed, Whisper-medium). Identity: higgs_tts3_ots.yaml at 42c12ad: model 'higgs_tts_3', request_model 'bosonai/higgs-tts-3-4b' = catalogue higgs-audio-v3-tts (model_repo bosonai/higgs-tts-3-4b); distinct from the paper's 'Higgs' row (Higgs Audio v2, already recorded).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "styletts-2-libritts",
    "bench": "rvcbench-libritts-ecapa-sim",
    "score": 0.23,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'StyleTTS2', column SIM = 0.23 (± printed; speaker-level cluster bootstrap, 1,000 resamples). Same authors, toolkit and clean LibriTTS condition as the v2 rows already recorded; the ten v2 systems other than CosyVoice print identical LibriTTS SIM/WER in v3. Identity: v3 now names the row 'StyleTTS2 [29] | 190M | Diffusion' in Table 5 and §3 lists StyleTTS2 among the 18 systems (v2 text said StyleTTS, the reason the row was excluded before); styletts2_ots.yaml at 42c12ad loads 'checkpoints/styletts2/StyleTTS2/Models/LibriTTS/epochs_2nd_00020.pth' with Models/LibriTTS/config.yml = the yl4579/StyleTTS2-LibriTTS multi-speaker checkpoint (catalogue styletts-2-libritts).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "styletts-2-libritts",
    "bench": "rvcbench-libritts-whisper-medium-wer",
    "score": 0.05,
    "source": "https://arxiv.org/html/2602.00443v3",
    "by": "independent",
    "date": "2026-10-01",
    "note": "RVCBench v3 (arXiv:2602.00443v3, 1 Oct 2026, CC BY 4.0) Table 13 'Performance on LibriTTS and VCTK' (caption: 'Appendix result tables cover the 18 representative main-paper models plus four extension systems: Fish Audio S2, MOSS-TTS v1.5, dots.tts, and Higgs TTS 3'), LibriTTS block, row 'StyleTTS2', column WER = 0.05 (fraction as printed, Whisper-medium). Identity: v3 now names the row 'StyleTTS2 [29] | 190M | Diffusion' in Table 5 and §3 lists StyleTTS2 among the 18 systems (v2 text said StyleTTS, the reason the row was excluded before); styletts2_ots.yaml at 42c12ad loads 'checkpoints/styletts2/StyleTTS2/Models/LibriTTS/epochs_2nd_00020.pth' with Models/LibriTTS/config.yml = the yl4579/StyleTTS2-LibriTTS multi-speaker checkpoint (catalogue styletts-2-libritts).",
    "source_id": "speech-rvcbench-ccby"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.03025,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'Scribe-V2', WER columns ES/EN 0.022, FR/EN 0.031, FR-CA/EN 0.041, DE/EN 0.027 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: ElevenLabs / Scribe-V2, Model ID 'scribe_v2', language_code omitted, defaults (Table 8) = catalogue elevenlabs-scribe-v2.",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "assemblyai-universal-3-5-pro",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.042,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'AssemblyAI', WER columns ES/EN 0.029, FR/EN 0.039, FR-CA/EN 0.052, DE/EN 0.048 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: Table 8 Model ID 'universal-3.5-pro' (language_detection: True, streaming: false); §4 text: 'AssemblyAI Universal-3.5-Pro' with link assemblyai.com/universal-3-pro. Table 8's provider label reads 'AssemblyAI / Universal-3-Pro', but the model ID and text both name 3.5-Pro = catalogue assemblyai-universal-3-5-pro.",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "voxtral-small-24b",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.06225,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'Voxtral', WER columns ES/EN 0.049, FR/EN 0.060, FR-CA/EN 0.059, DE/EN 0.081 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: Table 8: 'Mistral / Voxtral-Small-24B', 'Voxtral Small 1.0 (24B)', language omitted, temperature 0; cites Liu et al. (2025) Voxtral = catalogue voxtral-small-24b (2507).",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.08625,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'Parakeet', WER columns ES/EN 0.117, FR/EN 0.075, FR-CA/EN 0.099, DE/EN 0.054 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: Table 8: 'NVIDIA / Parakeet-TDT-0.6B-V3', model ID parakeet-tdt-0.6b-v3, defaults = catalogue parakeet-tdt-0-6b-v3.",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.05875,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'Nova-3', WER columns ES/EN 0.042, FR/EN 0.052, FR-CA/EN 0.072, DE/EN 0.069 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: Table 8: 'Deepgram / Nova-3-Multilang', model ID nova-3-multilang, language 'multi', smart_format true = the multilingual mode of catalogue deepgram-nova-3.",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "cose-e-eu4-mean-wer",
    "score": 0.43625,
    "source": "https://arxiv.org/html/2609.35645v1",
    "by": "independent",
    "date": "2026-09-28",
    "note": "CoSE-E (Gupta et al., ServiceNow AI Research / Qualcomm; arXiv:2609.35645v1, CC BY 4.0) Table 2 'WER, SWER, and AER by model and language pair', row 'Whisper', WER columns ES/EN 0.161, FR/EN 0.434, FR-CA/EN 0.535, DE/EN 0.615 (printed as fractions); score = unweighted mean of these four printed cells (derived, not printed). ZH/EN column not used. All models run by the authors with no language ID passed (Table 8). Identity: Table 8: 'OpenAI / Whisper-Large-V3-Turbo', model ID whisper-large-v3-turbo, language omitted, translate false = catalogue whisper-large-v3-turbo.",
    "invalid": "Source flags a decoding-mode failure, not an accuracy measurement: 'Run without a specified language ... it does not transcribe the mixed utterance at all: it detects a single language and translates the rest of the speech into English ... This is a decoding-mode failure ... so we report Whisper separately' (§5, Appendix D.3).",
    "source_id": "speech-stt-cose-e-ccby"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.0645,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'scribe_v2', WER (%) = 6.45 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'scribe-v2 | Unknown (API) | ElevenLabs | Commercial'; §2.3 'Scribe v2 [30], a commercial ElevenLabs API'. Shown as a proprietary reference (excluded from the paper's own best-value bolding, but measured identically).",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.0812,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'seamless-m4t-v2-large', WER (%) = 8.12 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'seamless-m4t-v2-large | w2v-BERT 2.0 + NLLB Decoder | ~2.3B | Meta | CC BY-NC 4.0'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "omniasr-llm-1b-v2",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.1664,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'omniASR_LLM_1B_v2', WER (%) = 16.64 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'omniASR-LLM-1B-v2 | Wav2Vec2 + LLM Decoder | ~1B | Meta' (v2 checkpoint, not v1).",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "whisper-large-v3",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.1934,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'whisper-large-v3', WER (%) = 19.34 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'whisper-large-v3 | Encoder-Decoder | ~1.5B | OpenAI'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "omniasr-ctc-1b-v2",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.2146,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'omniASR_CTC_1B_v2', WER (%) = 21.46 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'omniASR-CTC-1B-v2 | Wav2Vec2 + CTC Head | ~1B | Meta'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "voxtral-mini-3b",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.2332,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'Voxtral-Mini-3B-2507', WER (%) = 23.32 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'Voxtral-Mini-3B-2507 | Speech-LLM | ~3B | Mistral AI'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.3989,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'Qwen3-ASR-1.7B', WER (%) = 39.89 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'Qwen3-ASR-1.7B | Speech-LLM | ~1.7B | Alibaba'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.4847,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'Voxtral-Mini-4B-Realtime-2602', WER (%) = 48.47 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'Voxtral-Mini-4B-Realtime-2602 | Causal Speech-LLM | ~4B | Mistral AI'; run offline with the default delay setting (§2.4).",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.5185,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'parakeet-tdt-0.6b-v3', WER (%) = 51.85 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'parakeet-tdt-0.6b-v3 | FastConformer + TDT | ~600M | NVIDIA'. Its 25-language set excludes Basque, Catalan, Galician and Turkish (paper: 'Systems without explicit Basque coverage ... degrade to near-random output').",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "phi-4-multimodal-instruct",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.5631,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'Phi-4-multimodal-instruct', WER (%) = 56.31 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'Phi-4-multimodal-instruct | Multimodal-LLM | ~5.6B | Microsoft'.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "canary-1b-v2",
    "bench": "iberian-asr-7lang-macro-wer",
    "score": 0.6205,
    "source": "https://arxiv.org/html/2609.36920v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "López et al. (Telefónica Innovación Digital / UAM), arXiv:2609.36920v1 (CC BY 4.0) Table 3 'Overall ASR performance', row 'canary-1b-v2', WER (%) = 62.05 (macro-average over Basque, Catalan, Galician, Portuguese, Spanish, German, Turkish; per-language WER unweighted, Eq. 1). Run by the authors on one RTX 3090, sequential utterances, default settings. Identity: Table 2 'canary-1b-v2 | FastConformer + Decoder | ~978M | NVIDIA'. Same 25-language coverage caveat as parakeet-tdt-0.6b-v3.",
    "source_id": "speech-stt-iberian-bench-ccby"
  },
  {
    "model": "google-chirp-3",
    "bench": "mu-bench-batch-uer",
    "score": 0.119,
    "source": "https://arxiv.org/html/2609.32082v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "mu-bench (Li, Ray et al., UC Berkeley / Sierra AI; arXiv:2609.32082v1, CC BY 4.0) Table 5 'Results on mu-bench, sorted by UER (%)', row 'Google Chirp-3 | batch', column 'all' = 11.9 (UER macro-averaged over en-US, es-MX, tr-TR, vi-VN, zh-CN). Batch endpoint, 8 kHz mono PCM16 WAV, locale passed as language hint, other parameters default (§4.1); UER judged by gpt-6-astra (reasoning low) after reference-guided LLM normalization (reasoning high), seed 7. Identity: provider/product name only (paper gives no API model string); maps to catalogue google-chirp-3.",
    "source_id": "speech-stt-mu-bench-ccby"
  },
  {
    "model": "elevenlabs-scribe-v2",
    "bench": "mu-bench-batch-uer",
    "score": 0.173,
    "source": "https://arxiv.org/html/2609.32082v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "mu-bench (Li, Ray et al., UC Berkeley / Sierra AI; arXiv:2609.32082v1, CC BY 4.0) Table 5 'Results on mu-bench, sorted by UER (%)', row 'ElevenLabs Scribe v2 | batch', column 'all' = 17.3 (UER macro-averaged over en-US, es-MX, tr-TR, vi-VN, zh-CN). Batch endpoint, 8 kHz mono PCM16 WAV, locale passed as language hint, other parameters default (§4.1); UER judged by gpt-6-astra (reasoning low) after reference-guided LLM normalization (reasoning high), seed 7. Identity: provider/product name only (paper gives no API model string); maps to catalogue elevenlabs-scribe-v2.",
    "source_id": "speech-stt-mu-bench-ccby"
  },
  {
    "model": "gpt-4o-mini-transcribe",
    "bench": "mu-bench-batch-uer",
    "score": 0.272,
    "source": "https://arxiv.org/html/2609.32082v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "mu-bench (Li, Ray et al., UC Berkeley / Sierra AI; arXiv:2609.32082v1, CC BY 4.0) Table 5 'Results on mu-bench, sorted by UER (%)', row 'OpenAI GPT-4o Mini Transcribe | batch', column 'all' = 27.2 (UER macro-averaged over en-US, es-MX, tr-TR, vi-VN, zh-CN). Batch endpoint, 8 kHz mono PCM16 WAV, locale passed as language hint, other parameters default (§4.1); UER judged by gpt-6-astra (reasoning low) after reference-guided LLM normalization (reasoning high), seed 7. Identity: provider/product name only (paper gives no API model string); maps to catalogue gpt-4o-mini-transcribe.",
    "source_id": "speech-stt-mu-bench-ccby"
  },
  {
    "model": "deepgram-nova-3",
    "bench": "mu-bench-batch-uer",
    "score": 0.28,
    "source": "https://arxiv.org/html/2609.32082v1",
    "by": "independent",
    "date": "2026-09-25",
    "note": "mu-bench (Li, Ray et al., UC Berkeley / Sierra AI; arXiv:2609.32082v1, CC BY 4.0) Table 5 'Results on mu-bench, sorted by UER (%)', row 'Deepgram Nova-3 | batch', column 'all' = 28.0 (UER macro-averaged over en-US, es-MX, tr-TR, vi-VN, zh-CN). Batch endpoint, 8 kHz mono PCM16 WAV, locale passed as language hint, other parameters default (§4.1); UER judged by gpt-6-astra (reasoning low) after reference-guided LLM normalization (reasoning high), seed 7. Identity: provider/product name only (paper gives no API model string); maps to catalogue deepgram-nova-3.",
    "source_id": "speech-stt-mu-bench-ccby"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-en-wer",
    "score": 0.076,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '7.60(6.32)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model cosyvoice2-vc = FunAudioLLM/CosyVoice2-0.5B (same run config as the existing Seed-TTS rows).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.1391,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '13.91(11.96)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model cosyvoice2-vc = FunAudioLLM/CosyVoice2-0.5B (same run config as the existing Seed-TTS rows).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0835,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '8.35(12.58)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model cosyvoice2-vc = FunAudioLLM/CosyVoice2-0.5B (same run config as the existing Seed-TTS rows).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.7093,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '70.93(66.7)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: registry model cosyvoice2-vc = FunAudioLLM/CosyVoice2-0.5B (same run config as the existing Seed-TTS rows).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "cosyvoice2-0-5b",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0402,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/CosyVoice2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '4.02(4.08)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model cosyvoice2-vc = FunAudioLLM/CosyVoice2-0.5B (same run config as the existing Seed-TTS rows).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-en-wer",
    "score": 0.0501,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '5.01'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: cosyvoice3-latest config = FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint (not _RL), same as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.1105,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '11.05'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: cosyvoice3-latest config = FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint (not _RL), same as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0743,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '7.43'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: cosyvoice3-latest config = FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint (not _RL), same as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.7433,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '74.33'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: cosyvoice3-latest config = FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint (not _RL), same as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fun-cosyvoice3-0-5b-2512",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0406,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/CosyVoice3.md",
    "by": "independent",
    "date": "2025-12-17",
    "note": "UltraEval-Audio replication/CosyVoice3.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '4.06'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: cosyvoice3-latest config = FunAudioLLM/Fun-CosyVoice3-0.5B-2512 default checkpoint (not _RL), same as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-en-wer",
    "score": 0.0432,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '4.32'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model indextts2 = IndexTeam/IndexTTS-2.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0859,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '8.59'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model indextts2 = IndexTeam/IndexTTS-2.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0871,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '8.71'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model indextts2 = IndexTeam/IndexTTS-2.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.7401,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '74.01'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: registry model indextts2 = IndexTeam/IndexTTS-2.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "indextts-2",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0363,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/IndexTTS2.md",
    "by": "independent",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/IndexTTS2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.63'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model indextts2 = IndexTeam/IndexTTS-2.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-en-wer",
    "score": 0.0664,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '6.64(4.04)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model voxcpm-vc = openbmb/VoxCPM-0.5B; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0745,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '7.45(7.89)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry model voxcpm-vc = openbmb/VoxCPM-0.5B; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0995,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '9.95(12.9)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model voxcpm-vc = openbmb/VoxCPM-0.5B; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.6499,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '64.99(64.3)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: registry model voxcpm-vc = openbmb/VoxCPM-0.5B; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm-0-5b",
    "bench": "cv3-eval-zh-cer",
    "score": 0.034,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM.md",
    "by": "lab",
    "date": "2025-12-08",
    "note": "UltraEval-Audio replication/VoxCPM.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.40(3.40)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry model voxcpm-vc = openbmb/VoxCPM-0.5B; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-en-wer",
    "score": 0.055,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '5.50 (WER)'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), same run as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0639,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '6.39 (WER)'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), same run as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0839,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '8.39 (CER)'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), same run as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.6636,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '66.36'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), same run as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0393,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.93 (CER)'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B), same run as the existing Seed-TTS rows.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-en-wer",
    "score": 0.05278,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '5.278 (5.00)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: openbmb/VoxCPM2 (cfg_value=2.0, inference_timesteps=10, no denoising); OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.07222,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '7.222 (8.48)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: openbmb/VoxCPM2 (cfg_value=2.0, inference_timesteps=10, no denoising); OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.08327,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '8.327 (8.55)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: openbmb/VoxCPM2 (cfg_value=2.0, inference_timesteps=10, no denoising); OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.69055,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '69.055'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: openbmb/VoxCPM2 (cfg_value=2.0, inference_timesteps=10, no denoising); OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-zh-cer",
    "score": 0.03505,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.505 (3.65)'. First value is OpenBMB's reproduced measurement; the parenthesised official value is NOT used. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: openbmb/VoxCPM2 (cfg_value=2.0, inference_timesteps=10, no denoising); OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-en-wer",
    "score": 0.0377,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '3.77±0.19'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (registry qwen3-tts-1.7b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.079,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '7.90±1.77'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (registry qwen3-tts-1.7b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.1133,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '11.33±1.43'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (registry qwen3-tts-1.7b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.6606,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '66.06'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (registry qwen3-tts-1.7b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0312,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.12±0.07'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: row Qwen3-TTS-12Hz-1.7B-Base-official-infer-params (registry qwen3-tts-1.7b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.107,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '10.70±2.90'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params (registry qwen3-tts-0.6b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.107,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '10.70±1.06'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params (registry qwen3-tts-0.6b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.6704,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '67.04'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params (registry qwen3-tts-0.6b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "cv3-eval-zh-cer",
    "score": 0.034,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.40±0.09'. Mean ± spread as printed; the mean is recorded. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params (registry qwen3-tts-0.6b-base).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "cv3-eval-en-wer",
    "score": 0.0661,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_en (yuekai/CV3-Eval split zero_shot_en), column WER/CER: printed '6.61'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry/model/glmtts.yaml config glmtts: ckpt_dir zai-org/GLM-TTS, use_phoneme False (released base checkpoint; the RL weights are 'coming soon' per the GLM-TTS README).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.094,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column WER/CER: printed '9.40 (note 'fail_rate: 3.12%')'. Scorer per registry/evaluator/cv3.yaml: openai Whisper large-v3 (registry cv3-whisper: path large-v3). Identity: registry/model/glmtts.yaml config glmtts: ckpt_dir zai-org/GLM-TTS, use_phoneme False (released base checkpoint; the RL weights are 'coming soon' per the GLM-TTS README).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0914,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_zh (yuekai/CV3-Eval split zero_shot_hard_zh), column WER/CER: printed '9.14'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry/model/glmtts.yaml config glmtts: ckpt_dir zai-org/GLM-TTS, use_phoneme False (released base checkpoint; the RL weights are 'coming soon' per the GLM-TTS README).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.7343,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_hard_en (yuekai/CV3-Eval split zero_shot_hard_en), column SIM: printed '73.43'. Scorer per registry/evaluator/cv3.yaml: cv3-speaker-sim: speech_eres2net_sv_en_voxceleb_16k (ERes2Net SV). Identity: registry/model/glmtts.yaml config glmtts: ckpt_dir zai-org/GLM-TTS, use_phoneme False (released base checkpoint; the RL weights are 'coming soon' per the GLM-TTS README).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "cv3-eval-zh-cer",
    "score": 0.0361,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, CV3 Benchmark (Zero-Shot) table, dataset cv3_zero_shot_zh (yuekai/CV3-Eval split zero_shot_zh), column WER/CER: printed '3.61'. Scorer per registry/evaluator/cv3.yaml: Seaco Paraformer large zh (speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch). Identity: registry/model/glmtts.yaml config glmtts: ckpt_dir zai-org/GLM-TTS, use_phoneme False (released base checkpoint; the RL weights are 'coming soon' per the GLM-TTS README).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0248,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, Seed-TTS-Eval Benchmark table, row seed_tts_eval_en WER/CER: printed '2.48'. Same scorers as the existing UltraEval Seed-TTS rows (registry/evaluator/seed_tts_eval.yaml: openai/whisper-large-v3). Identity: registry/model/glmtts.yaml config glmtts, ckpt_dir zai-org/GLM-TTS, use_phoneme False.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0106,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, Seed-TTS-Eval Benchmark table, row seed_tts_eval_zh WER/CER: printed '1.06(1.03)' (first value is OpenBMB's reproduced measurement; parenthesised official value NOT used). Same scorers as the existing UltraEval Seed-TTS rows (registry/evaluator/seed_tts_eval.yaml: Seaco Paraformer zh). Identity: registry/model/glmtts.yaml config glmtts, ckpt_dir zai-org/GLM-TTS, use_phoneme False.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.6725,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, Seed-TTS-Eval Benchmark table, row seed_tts_eval_en SIM: printed '67.25'. Same scorers as the existing UltraEval Seed-TTS rows (registry/evaluator/seed_tts_eval.yaml: WavLM-large SV via audio_evals/lib/simo). Identity: registry/model/glmtts.yaml config glmtts, ckpt_dir zai-org/GLM-TTS, use_phoneme False.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "glm-tts",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.7598,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/GLM-TTS.md",
    "by": "independent",
    "date": "2025-12-01",
    "note": "UltraEval-Audio replication/GLM-TTS.md, Seed-TTS-Eval Benchmark table, row seed_tts_eval_zh SIM: printed '75.98(76.1)' (first value is OpenBMB's reproduced measurement; parenthesised official value NOT used). Same scorers as the existing UltraEval Seed-TTS rows (registry/evaluator/seed_tts_eval.yaml: WavLM-large SV via audio_evals/lib/simo). Identity: registry/model/glmtts.yaml config glmtts, ckpt_dir zai-org/GLM-TTS, use_phoneme False.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fasih-tts-v1",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1049.8,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id fasih_tts_v1 'Fasih TTS V1': elo 1049.8 ±22.1, 552 battles (W134/L170/T248), active (not retired). Same pinned snapshot as the existing 10 rows; the 2026-10-09 snapshot (5cebcf3) has no new models and is not mixed in. Identity: NightPrince/Fasih-TTS-V1 (arena wrapper models/fasih_tts_v1.py loads MODEL_REPO NightPrince/Fasih-TTS-V1 with its shipped speaker latents).",
    "source_id": "speech-arabic-tts-arena-apache"
  },
  {
    "model": "silma-tts-v1",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 1006.7,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id silma_tts_v1_small 'SILMA TTS v1 Small': elo 1006.7 ±17.4, 1038 battles (W256/L329/T453), active (not retired). Same pinned snapshot as the existing 10 rows; the 2026-10-09 snapshot (5cebcf3) has no new models and is not mixed in. Identity: silma-ai/silma-tts via the silma-tts pip package (arena wrapper models/silma_tts_v1_small.py; model_url https://huggingface.co/silma-ai/silma-tts). Not 'SILMA TTS v1 Large' (retired, commercial) or 'SILMA TTS v2 MSA' (commercial).",
    "source_id": "speech-arabic-tts-arena-apache"
  },
  {
    "model": "nabra-82m-v0-1",
    "bench": "arabic-tts-arena-elo-20260924",
    "score": 965,
    "source": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json",
    "by": "independent",
    "date": "2026-09-24",
    "note": "leaderboard.json (last_updated 2026-09-24T00:01:34Z, 8,647 total votes) model_id nabra_82m 'Nabra 82M': elo 965.0 ±23.9, 508 battles (W56/L218/T234), active (not retired). Same pinned snapshot as the existing 10 rows; the 2026-10-09 snapshot (5cebcf3) has no new models and is not mixed in. Identity: oddadmix/Nabra-82M-v0.1 pinned by the arena wrapper models/nabra_82m.py to MODEL_REVISION adf6abf35c46db5f2b08803b5067a12c759b1ee1 (= current HF sha).",
    "source_id": "speech-arabic-tts-arena-apache"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "minimax-ml21-ultraeval-macro-err",
    "score": 0.080557,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/fishaudio-s2-pro.md",
    "by": "independent",
    "date": "2026-04-01",
    "note": "UltraEval-Audio replication/fishaudio-s2-pro.md, 'MiniMax TTS Multilingual Benchmark' table (dataset MiniMaxAI/TTS-Multilingual-Test-Set), WER/CER column, reproduced first value only (parenthesised official values NOT used). Unweighted mean over the 21 languages all four UltraEval runs report (Indonesian, Italian, Ukrainian left out because at least one run lacks them) = 8.0557%. Per-language cells: arabic 8.02 (WER); cantonese 46.15 (CER); chinese 1.08 (CER); czech 6.57 (WER); dutch 1.40 (WER); english 2.39 (WER); finnish 7.69 (WER); french 4.18 (WER); german 1.04 (WER); greek 12.45 (WER); hindi 22.38 (WER); japanese 3.65 (CER); korean 1.59 (CER); polish 2.41 (WER); portuguese 1.53 (WER); romanian 15.83 (WER); russian 3.89 (WER); spanish 1.56 (WER); thai 7.59 (CER); turkish 2.03 (WER); vietnamese 15.74 (WER). Scorer registry/evaluator/minimax.yaml: seed-tts-whisper (Whisper large-v3) for every language except Chinese (Seaco Paraformer large); CER for zh/yue/ja/ko/th as printed. Identity: fishaudio/s2-pro open weights with the fish-speech local inference path (doc note: not Fish Audio's online service).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "fireredtts3-base",
    "bench": "minimax-ml21-ultraeval-macro-err",
    "score": 0.054233,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/FireRedTTS3.md",
    "by": "independent",
    "date": "2026-08-22",
    "note": "UltraEval-Audio replication/FireRedTTS3.md, 'MiniMax TTS Multilingual Benchmark' table (dataset MiniMaxAI/TTS-Multilingual-Test-Set), WER/CER column, reproduced first value only (parenthesised official values NOT used). Unweighted mean over the 21 languages all four UltraEval runs report (Indonesian, Italian, Ukrainian left out because at least one run lacks them) = 5.4233%. Per-language cells: arabic 12.13 (WER, —); cantonese 39.80 (40.32); chinese 0.94 (0.91); czech 3.53 (3.17); dutch 0.98 (1.15); english 2.17 (2.12); finnish 3.29 (3.10); french 5.85 (5.28); german 0.40 (0.69); greek 3.03 (WER, —); hindi 19.41 (WER, —); japanese 3.45 (3.60); korean 2.52 (2.42); polish 0.99 (1.22); portuguese 1.73 (1.79); romanian 2.62 (1.93); russian 5.03 (3.28); spanish 1.15 (1.21); thai 2.16 (1.87); turkish 1.39 (0.92); vietnamese 1.32 (WER, —). Scorer registry/evaluator/minimax.yaml: seed-tts-whisper (Whisper large-v3) for every language except Chinese (Seaco Paraformer large); CER for zh/yue/ja/ko/th as printed. Identity: FireRedTeam/FireRedTTS3 base (zero-shot cloning) config fireredtts3-base, explicit --prompt fireredtts3-voice-clone-<language>.",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "minimax-ml21-ultraeval-macro-err",
    "score": 0.059471,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/moss-tts-v1.5.md",
    "by": "independent",
    "date": "2026-08-03",
    "note": "UltraEval-Audio replication/moss-tts-v1.5.md, 'MiniMax TTS Multilingual Benchmark' table (dataset MiniMaxAI/TTS-Multilingual-Test-Set), WER/CER column, reproduced first value only (parenthesised official values NOT used). Unweighted mean over the 21 languages all four UltraEval runs report (Indonesian, Italian, Ukrainian left out because at least one run lacks them) = 5.9471%. Per-language cells: arabic 10.25 (WER); cantonese 35.25 (CER); chinese 1.23 (CER); czech 5.42 (WER); dutch 1.22 (WER); english 2.20 (WER); finnish 8.60 (WER); french 5.43 (WER); german 0.55 (WER); greek 4.50 (WER); hindi 20.86 (WER); japanese 3.82 (CER); korean 2.20 (CER); polish 2.67 (WER); portuguese 2.33 (WER); romanian 4.05 (WER); russian 5.24 (WER); spanish 1.30 (WER); thai 2.82 (CER); turkish 2.07 (WER); vietnamese 2.88 (WER). Scorer registry/evaluator/minimax.yaml: seed-tts-whisper (Whisper large-v3) for every language except Chinese (Seaco Paraformer large); CER for zh/yue/ja/ko/th as printed. Identity: OpenMOSS-Team/MOSS-TTS-v1.5 (MossTTSDelay-8B).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "voxcpm2",
    "bench": "minimax-ml21-ultraeval-macro-err",
    "score": 0.071357,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/VoxCPM2.md",
    "by": "lab",
    "date": "2026-07-27",
    "note": "UltraEval-Audio replication/VoxCPM2.md, 'MiniMax TTS Multilingual Benchmark' table (dataset MiniMaxAI/TTS-Multilingual-Test-Set), WER/CER column, reproduced first value only (parenthesised official values NOT used). Unweighted mean over the 21 languages all four UltraEval runs report (Indonesian, Italian, Ukrainian left out because at least one run lacks them) = 7.1357%. Per-language cells: arabic 12.797 (13.046); cantonese 38.334 (38.584); chinese 1.040 (1.136); czech 24.567 (24.132); dutch 0.740 (0.913); english 2.371 (2.289); finnish 2.458 (2.632); french 4.515 (4.534); german 1.054 (0.679); greek 2.660 (2.844); hindi 17.968 (19.699); japanese 4.201 (4.628); korean 3.651 (1.962); polish 1.379 (1.141); portuguese 2.036 (1.938); romanian 20.772 (21.577); russian 3.035 (3.634); spanish 1.482 (1.438); thai 2.215 (2.961); turkish 0.611 (0.817); vietnamese 1.963 (3.307). Scorer registry/evaluator/minimax.yaml: seed-tts-whisper (Whisper large-v3) for every language except Chinese (Seaco Paraformer large); CER for zh/yue/ja/ko/th as printed. Identity: openbmb/VoxCPM2, cfg_value=2.0, inference_timesteps=10; OpenBMB evaluates its own model (by=lab).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "parakeet-ultra",
    "bench": "librispeech-test-clean-wer",
    "score": 0.018,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-ultra.json",
    "by": "independent",
    "date": "2026-10-05",
    "note": "transcribe.cpp catalog/parakeet-ultra.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.8% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.65, 1.95], batch_size 8, timestamps none, backend cuda, engine 021aa17d, profile asr-publication-v2, empty_hyp 0. Upstream moondream/parakeet-ultra@73175eb converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.8, 'Q8_0': 1.8, 'Q6_K': 1.81, 'Q5_K_M': 1.81, 'Q4_K_M': 1.82}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-ultra",
    "bench": "fleurs-en-wer",
    "score": 0.0426,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-ultra.json",
    "by": "independent",
    "date": "2026-10-05",
    "note": "transcribe.cpp catalog/parakeet-ultra.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.26% at Q8_0 (95% CI [3.84, 4.69]; batch_size 8, timestamps none, language_hint en). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "orukeet",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0186,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/orukeet.json",
    "by": "independent",
    "date": "2026-10-06",
    "note": "transcribe.cpp catalog/orukeet.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.86% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.71, 2.01], batch_size 8, timestamps none, backend cuda, engine 37273409, profile asr-publication-v2, empty_hyp 0. Upstream oruk/orukeet@b59c13a converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.86, 'Q8_0': 1.86, 'Q6_K': 1.86, 'Q5_K_M': 1.83, 'Q4_K_M': 1.95}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "orukeet",
    "bench": "fleurs-en-wer",
    "score": 0.0443,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/orukeet.json",
    "by": "independent",
    "date": "2026-10-06",
    "note": "transcribe.cpp catalog/orukeet.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.43% at Q8_0 (95% CI [3.99, 4.9]; batch_size 8, timestamps none, language_hint en). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-unified-en-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0159,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-unified-en-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-unified-en-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.59% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.45, 1.73], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream nvidia/parakeet-unified-en-0.6b@d4ac992 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.59, 'Q8_0': 1.6, 'Q6_K': 1.61, 'Q5_K_M': 1.58, 'Q4_K_M': 1.62}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-unified-en-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0399,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-unified-en-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-unified-en-0.6b.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 3.99% at Q8_0 (95% CI [3.6, 4.42]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "nemotron-speech-streaming-en-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0231,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/nemotron-speech-streaming-en-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/nemotron-speech-streaming-en-0.6b.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 2.31% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [2.13, 2.49], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream nvidia/nemotron-speech-streaming-en-0.6b@ef3bf40 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 2.31, 'Q8_0': 2.31, 'Q6_K': 2.29, 'Q5_K_M': 2.34, 'Q4_K_M': 2.38}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "nemotron-speech-streaming-en-0-6b",
    "bench": "fleurs-en-wer",
    "score": 0.0643,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/nemotron-speech-streaming-en-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/nemotron-speech-streaming-en-0.6b.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 6.43% at Q8_0 (95% CI [5.81, 7.14]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "multitalker-parakeet-streaming-0-6b-v1",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0219,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/multitalker-parakeet-streaming-0.6b-v1.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/multitalker-parakeet-streaming-0.6b-v1.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 2.19% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [2.03, 2.37], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream nvidia/multitalker-parakeet-streaming-0.6b-v1@8749fc7 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 2.19, 'Q8_0': 2.18, 'Q6_K': 2.2, 'Q5_K_M': 2.18, 'Q4_K_M': 2.18}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "multitalker-parakeet-streaming-0-6b-v1",
    "bench": "fleurs-en-wer",
    "score": 0.0652,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/multitalker-parakeet-streaming-0.6b-v1.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/multitalker-parakeet-streaming-0.6b-v1.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 6.52% at Q8_0 (95% CI [5.95, 7.12]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-ctc-1-1b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0187,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-tdt_ctc-1.1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt_ctc-1.1b.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.87% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.72, 2.02], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream nvidia/parakeet-tdt_ctc-1.1b@675e786 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.87, 'Q8_0': 1.87, 'Q6_K': 1.87, 'Q5_K_M': 1.87, 'Q4_K_M': 1.91}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-tdt-ctc-1-1b",
    "bench": "fleurs-en-wer",
    "score": 0.0468,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-tdt_ctc-1.1b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-tdt_ctc-1.1b.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.68% at Q8_0 (95% CI [4.22, 5.15]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-4-1-2b-nar",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0129,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-4.1-2b-nar.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b-nar.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.29% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.15, 1.44], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream ibm-granite/granite-speech-4.1-2b-nar@99a4df9 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.29, 'Q8_0': 1.29, 'Q6_K': 1.29, 'Q5_K_M': 1.28, 'Q4_K_M': 1.34}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-4-1-2b-nar",
    "bench": "fleurs-en-wer",
    "score": 0.0533,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-4.1-2b-nar.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b-nar.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 5.33% at Q8_0 (95% CI [4.6, 6.16]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-4-1-2b-plus",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0149,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-4.1-2b-plus.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b-plus.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.49% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.35, 1.64], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 0. Upstream ibm-granite/granite-speech-4.1-2b-plus@edd3bf5 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.48, 'Q8_0': 1.5, 'Q6_K': 1.46, 'Q5_K_M': 1.48, 'Q4_K_M': 1.56}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-4-1-2b-plus",
    "bench": "fleurs-en-wer",
    "score": 0.0446,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-4.1-2b-plus.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-4.1-2b-plus.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.46% at Q8_0 (95% CI [3.97, 4.94]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-5-0-470m-turboctc-nc",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0129,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-5.0-470m-turboctc-nc.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-5.0-470m-turboctc-nc.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.29% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.17, 1.44], batch_size 8, timestamps none, backend cuda, engine 9daf396, profile asr-publication-v2, empty_hyp 0. Upstream ibm-granite/granite-speech-5.0-470m-turboctc-nc@0eb7b4f converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.28, 'Q8_0': 1.29, 'Q6_K': 1.29, 'Q5_K_M': 1.29, 'Q4_K_M': 1.34}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "granite-speech-5-0-470m-turboctc-nc",
    "bench": "fleurs-en-wer",
    "score": 0.043,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/granite-speech-5.0-470m-turboctc-nc.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/granite-speech-5.0-470m-turboctc-nc.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.3% at Q8_0 (95% CI [3.86, 4.76]; batch_size 8, timestamps none, language_hint en). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moss-transcribe-diarize",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0208,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/moss-transcribe-diarize.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moss-transcribe-diarize.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 2.08% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [1.83, 2.4], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 3. Upstream OpenMOSS-Team/MOSS-Transcribe-Diarize@d7231bb converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 2.07, 'Q8_0': 1.93, 'Q6_K': 1.96, 'Q5_K_M': 1.99, 'Q4_K_M': 2.59}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moss-transcribe-diarize",
    "bench": "fleurs-en-wer",
    "score": 0.0513,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/moss-transcribe-diarize.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moss-transcribe-diarize.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 5.13% at Q8_0 (95% CI [4.57, 5.78]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "moss-transcribe-diarize",
    "bench": "fleurs-zh-err",
    "score": 0.0923,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/moss-transcribe-diarize.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/moss-transcribe-diarize.json accuracy_benchmarks: FLEURS test, language zh (FLEURS cmn_hans_cn per scripts/wer/languages.py; CER, whitespace stripped, BasicTextNormalizer per scripts/wer/score.py) (945 utt.), corpus CER 9.23% at Q8_0 (95% CI [8.2, 10.17]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0178,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-nano-2512.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.78% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [None, None], batch_size None, timestamps None, backend None, engine None, profile None, empty_hyp None. Upstream FunAudioLLM/Fun-ASR-Nano-2512@a7088d6 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.79, 'Q8_0': 1.79, 'Q6_K': 1.78, 'Q5_K_M': 1.82, 'Q4_K_M': 1.92}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "fleurs-en-wer",
    "score": 0.0549,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-nano-2512.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 5.49% at Q8_0 (95% CI [4.91, 6.04]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "fleurs-zh-err",
    "score": 0.0859,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-nano-2512.json accuracy_benchmarks: FLEURS test, language zh (FLEURS cmn_hans_cn per scripts/wer/languages.py; CER, whitespace stripped, BasicTextNormalizer per scripts/wer/score.py) (945 utt.), corpus CER 8.59% at Q8_0 (95% CI [7.59, 9.53]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-mlt-nano-2512",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0174,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-mlt-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-mlt-nano-2512.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 1.74% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [None, None], batch_size None, timestamps None, backend None, engine None, profile None, empty_hyp None. Upstream FunAudioLLM/Fun-ASR-MLT-Nano-2512@cf67a93 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 1.74, 'Q8_0': 1.74, 'Q6_K': 1.69, 'Q5_K_M': 1.77, 'Q4_K_M': 1.89}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-mlt-nano-2512",
    "bench": "fleurs-en-wer",
    "score": 0.049,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-mlt-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-mlt-nano-2512.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.9% at Q8_0 (95% CI [4.4, 5.48]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "fun-asr-mlt-nano-2512",
    "bench": "fleurs-zh-err",
    "score": 0.0864,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/fun-asr-mlt-nano-2512.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/fun-asr-mlt-nano-2512.json accuracy_benchmarks: FLEURS test, language zh (FLEURS cmn_hans_cn per scripts/wer/languages.py; CER, whitespace stripped, BasicTextNormalizer per scripts/wer/score.py) (945 utt.), corpus CER 8.64% at Q8_0 (95% CI [7.7, 9.55]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "sensevoice-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0313,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/sensevoice-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/sensevoice-small.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 3.13% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [None, None], batch_size None, timestamps None, backend None, engine None, profile None, empty_hyp None. Upstream FunAudioLLM/SenseVoiceSmall@3eb3b4e converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 3.13, 'Q8_0': 3.13, 'Q6_K': 3.14, 'Q5_K_M': 3.18, 'Q4_K_M': 3.45}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "sensevoice-small",
    "bench": "fleurs-en-wer",
    "score": 0.0714,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/sensevoice-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/sensevoice-small.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 7.14% at Q8_0 (95% CI [6.54, 7.77]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "sensevoice-small",
    "bench": "fleurs-zh-err",
    "score": 0.1012,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/sensevoice-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/sensevoice-small.json accuracy_benchmarks: FLEURS test, language zh (FLEURS cmn_hans_cn per scripts/wer/languages.py; CER, whitespace stripped, BasicTextNormalizer per scripts/wer/score.py) (945 utt.), corpus CER 10.12% at Q8_0 (95% CI [9.16, 11.08]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "breeze-asr-25",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0229,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/breeze-asr-25.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/breeze-asr-25.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 2.29% at BF16 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [None, None], batch_size 1, timestamps segment, backend None, engine None, profile None, empty_hyp None. Upstream MediaTek-Research/Breeze-ASR-25@cffe7cc converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 2.29, 'Q8_0': 2.27, 'Q6_K': 2.29, 'Q5_K_M': 2.25, 'Q4_K_M': 2.26}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "breeze-asr-25",
    "bench": "fleurs-en-wer",
    "score": 0.0414,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/breeze-asr-25.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/breeze-asr-25.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.14% at Q8_0 (95% CI [3.7, 4.61]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "breeze-asr-25",
    "bench": "fleurs-zh-err",
    "score": 0.081,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/breeze-asr-25.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/breeze-asr-25.json accuracy_benchmarks: FLEURS test, language zh (FLEURS cmn_hans_cn per scripts/wer/languages.py; CER, whitespace stripped, BasicTextNormalizer per scripts/wer/score.py) (945 utt.), corpus CER 8.1% at Q8_0 (95% CI [7.28, 8.94]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source). Other tiers in the file: {'BF16': 8.12, 'F16': 8.11, 'Q6_K': 8.12, 'Q5_K_M': 8.12, 'Q4_K_M': 8.08}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "medasr",
    "bench": "librispeech-test-clean-wer",
    "score": 0.1788,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/medasr.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/medasr.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 17.88% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [17.34, 18.45], batch_size 1, timestamps none, backend None, engine None, profile None, empty_hyp 14. Upstream google/medasr@ae1e484 converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'F16': 17.88, 'Q8_0': 17.86, 'Q6_K': 17.93, 'Q5_K_M': 17.91, 'Q4_K_M': 18.14}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "medasr",
    "bench": "fleurs-en-wer",
    "score": 0.3748,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/medasr.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/medasr.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 37.48% at Q8_0 (95% CI [35.83, 39.09]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-primeline",
    "bench": "librispeech-test-clean-wer",
    "score": 0.022,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-primeline.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-primeline.json accuracy_benchmarks: LibriSpeech test-clean (2620 utt.) corpus WER 2.2% at F32 (full-precision tier; F32 > BF16 > F16 preference as for the existing rows), 95% CI [2.04, 2.37], batch_size 1, timestamps none, backend cuda, engine bcd4a0f, profile None, empty_hyp 0. Upstream primeline/parakeet-primeline@3f1a9bc converted to GGUF and validated against the reference implementation by the transcribe.cpp authors; recipe greedy, language forced to en, whisper_normalizer EnglishTextNormalizer + jiwer (docs/tools/wer.md). Quantized rows (not used): {'Q8_0': 2.2, 'F16': 2.19, 'Q6_K': 2.19, 'Q5_K_M': 2.2, 'Q4_K_M': 2.26}.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "parakeet-primeline",
    "bench": "fleurs-en-wer",
    "score": 0.0482,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/18947cf405f957195dd457db4e34c1cff9e88889/catalog/parakeet-primeline.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/parakeet-primeline.json accuracy_benchmarks: FLEURS test, language en (en_us; EnglishTextNormalizer + jiwer) (647 utt.), corpus WER 4.82% at Q8_0 (95% CI [4.39, 5.34]; batch_size 8, timestamps none, language_hint None). FLEURS is benchmarked at Q8_0 in the catalog's asr-publication-v2 profile (same convention as the existing fleurs-en-wer rows from this source).",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "cohere-transcribe-03-2026",
    "bench": "fleurs-zh-err",
    "score": 0.1118,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/cohere-transcribe-03-2026.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/cohere-transcribe-03-2026.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 11.18% at Q8_0 (945 utt., 95% CI [10.19, 12.21], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream CohereLabs/cohere-transcribe-03-2026@76b8b23.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "fleurs-zh-err",
    "score": 0.1887,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/nemotron-3.5-asr-streaming-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/nemotron-3.5-asr-streaming-0.6b.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 18.87% at Q8_0 (945 utt., 95% CI [17.82, 19.91], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream nvidia/nemotron-3.5-asr-streaming-0.6b@24b151a.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "fleurs-zh-err",
    "score": 0.0757,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-0.6b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/qwen3-asr-0.6b.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 7.57% at Q8_0 (945 utt., 95% CI [6.7, 8.42], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream Qwen/Qwen3-ASR-0.6B@5eb1441.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "fleurs-zh-err",
    "score": 0.0714,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/qwen3-asr-1.7b.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/qwen3-asr-1.7b.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 7.14% at Q8_0 (945 utt., 95% CI [6.26, 8.12], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream Qwen/Qwen3-ASR-1.7B@7278e1e.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "fleurs-zh-err",
    "score": 0.1041,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/voxtral-mini-4b-realtime-2602.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/voxtral-mini-4b-realtime-2602.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 10.41% at Q8_0 (945 utt., 95% CI [9.32, 11.52], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream mistralai/Voxtral-Mini-4B-Realtime-2602@2769294.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-base",
    "bench": "fleurs-zh-err",
    "score": 0.3621,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-base.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-base.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 36.21% at Q8_0 (945 utt., 95% CI [35.11, 37.29], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-base@e37978b.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v2",
    "bench": "fleurs-zh-err",
    "score": 0.1539,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v2.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v2.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 15.39% at Q8_0 (945 utt., 95% CI [14.02, 16.64], batch_size 8, language_hint zh). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-large-v2@ae46427.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "fleurs-zh-err",
    "score": 0.085,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3-turbo.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v3-turbo.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 8.5% at Q8_0 (945 utt., 95% CI [7.65, 9.4], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-large-v3-turbo@41f01f3.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-zh-err",
    "score": 0.0798,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large-v3.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large-v3.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 7.98% at Q8_0 (945 utt., 95% CI [7.12, 8.82], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-large-v3@06f233f.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-large",
    "bench": "fleurs-zh-err",
    "score": 0.1932,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-large.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-large.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 19.32% at Q8_0 (945 utt., 95% CI [18.03, 20.64], batch_size 8, language_hint zh). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-large@4ef9b41.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-medium",
    "bench": "fleurs-zh-err",
    "score": 0.1313,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-medium.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-medium.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 13.13% at Q8_0 (945 utt., 95% CI [11.97, 14.25], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-medium@abdf7c3.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-small",
    "bench": "fleurs-zh-err",
    "score": 0.2306,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-small.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-small.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 23.06% at Q8_0 (945 utt., 95% CI [21.79, 24.35], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-small@973afd2.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "whisper-tiny",
    "bench": "fleurs-zh-err",
    "score": 0.4191,
    "source": "https://github.com/handy-computer/transcribe.cpp/blob/c1fee503219ea82dc2e42917150198a31b2ecc2a/catalog/whisper-tiny.json",
    "by": "independent",
    "date": "2026-09-15",
    "note": "transcribe.cpp catalog/whisper-tiny.json (same pinned file as this model's existing LibriSpeech/FLEURS-en rows) accuracy_benchmarks: FLEURS test, language zh = cmn_hans_cn (scripts/wer/languages.py), corpus CER 41.91% at Q8_0 (945 utt., 95% CI [40.85, 42.93], batch_size 8, language_hint None). CER with whitespace stripped and BasicTextNormalizer (scripts/wer/score.py). FLEURS is benchmarked only at Q8_0 in the asr-publication-v2 profile. Upstream openai/whisper-tiny@169d4a4.",
    "source_id": "speech-stt-transcribecpp-mit"
  },
  {
    "model": "inworld-stt-1",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0171,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'inworld-stt-1': corpus WER 1.71% (95% CI 1.57-1.87), scored 2618 of 2620 utterances (99.9% coverage), 52924 reference words. Inworld /stt/v1/transcribe with transcribeConfig.modelId 'inworld/inworld-stt-1' (adapter registry model 'inworld/inworld-stt-1').",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "fish-audio-transcribe-1",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0173,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'fish-audio-asr': corpus WER 1.73% (95% CI 1.52-1.96), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Fish Audio /v1/asr with no model header (adapter comment: 'single model transcribe-1; the /v1/asr endpoint takes no model param'); Fish docs: a request whose model header is missing is served and billed as transcribe-1.",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "grok-voice-transcribe-1-0",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0182,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'grok-stt': corpus WER 1.82% (95% CI 1.66-1.99), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. xAI /v1/stt with no model field (adapter registry label 'grok-stt'). Runs were 2026-07-16..18 (README); xAI release notes: grok-voice-transcribe-2.0 first became available in September 2026 ('Use grok-voice-transcribe-1.0 or grok-voice-transcribe-2.0; the default is grok-voice-transcribe-2.0'), so in July the only and default xAI STT model was grok-voice-transcribe-1.0 (identity inferred from dates, not printed by verli).",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "cartesia-ink-whisper",
    "bench": "verli-librispeech-clean-wer",
    "score": 0.0201,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'cartesia-ink-whisper': corpus WER 2.01% (95% CI 1.83-2.21), scored 2620 of 2620 utterances (100.0% coverage), 53027 reference words. Cartesia batch /stt with model 'ink-whisper' (adapter registry; 'Batch endpoint only serves the ink-whisper family').",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "grok-voice-transcribe-1-0",
    "bench": "verli-earnings22-wer",
    "score": 0.0953,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'grok-stt': corpus WER 9.53% (95% CI 9.09-9.96), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. xAI /v1/stt with no model field (adapter registry label 'grok-stt'). Runs were 2026-07-16..18 (README); xAI release notes: grok-voice-transcribe-2.0 first became available in September 2026 ('Use grok-voice-transcribe-1.0 or grok-voice-transcribe-2.0; the default is grok-voice-transcribe-2.0'), so in July the only and default xAI STT model was grok-voice-transcribe-1.0 (identity inferred from dates, not printed by verli).",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "fish-audio-transcribe-1",
    "bench": "verli-earnings22-wer",
    "score": 0.1027,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'fish-audio-asr': corpus WER 10.27% (95% CI 9.86-10.71), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Fish Audio /v1/asr with no model header (adapter comment: 'single model transcribe-1; the /v1/asr endpoint takes no model param'); Fish docs: a request whose model header is missing is served and billed as transcribe-1.",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "inworld-stt-1",
    "bench": "verli-earnings22-wer",
    "score": 0.1029,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'inworld-stt-1': corpus WER 10.29% (95% CI 9.86-10.74), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Inworld /stt/v1/transcribe with transcribeConfig.modelId 'inworld/inworld-stt-1' (adapter registry model 'inworld/inworld-stt-1').",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "cartesia-ink-whisper",
    "bench": "verli-earnings22-wer",
    "score": 0.1159,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'cartesia-ink-whisper': corpus WER 11.59% (95% CI 11.12-12.09), scored 2741 of 2741 utterances (100.0% coverage), 48899 reference words. Cartesia batch /stt with model 'ink-whisper' (adapter registry; 'Batch endpoint only serves the ink-whisper family').",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "fish-audio-transcribe-1",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.0261,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'fish-audio-asr': corpus WER 2.61% (95% CI 2.39-2.84), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. Fish Audio /v1/asr with no model header (adapter comment: 'single model transcribe-1; the /v1/asr endpoint takes no model param'); Fish docs: a request whose model header is missing is served and billed as transcribe-1.",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "grok-voice-transcribe-1-0",
    "bench": "verli-spgispeech1200-wer",
    "score": 0.038,
    "source": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json",
    "by": "independent",
    "date": "2026-07-19",
    "note": "engine 'grok-stt': corpus WER 3.8% (95% CI 3.47-4.17), scored 1200 of 1200 utterances (100.0% coverage), 29400 reference words. xAI /v1/stt with no model field (adapter registry label 'grok-stt'). Runs were 2026-07-16..18 (README); xAI release notes: grok-voice-transcribe-2.0 first became available in September 2026 ('Use grok-voice-transcribe-1.0 or grok-voice-transcribe-2.0; the default is grok-voice-transcribe-2.0'), so in July the only and default xAI STT model was grok-voice-transcribe-1.0 (identity inferred from dates, not printed by verli).",
    "source_id": "speech-stt-verli-mit"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0109,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-3.5B', column 'ZH CER (%)' = 1.09 (% / 100). Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.818,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-3.5B', column 'ZH SIM' = 0.818. Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.015,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-3.5B', column 'EN WER (%)' = 1.5 (% / 100). Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.786,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-3.5B', column 'EN SIM' = 0.786. Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-1b",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0118,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-1B', column 'ZH CER (%)' = 1.18 (% / 100). Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-1b",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.812,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-1B', column 'ZH SIM' = 0.812. Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-1b",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0178,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-1B', column 'EN WER (%)' = 1.78 (% / 100). Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-1b",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.762,
    "source": "https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/README.md",
    "by": "lab",
    "date": "2026-04-03",
    "note": "Pinned LongCat-AudioDiT-3.5B model card (MIT), 'Experimental Results on Seed Benchmark' table, row 'LongCat-AudioDiT-1B', column 'EN SIM' = 0.762. Setup (arXiv:2603.29339v1 §5.1, same authors): Whisper large-v3 for English and Paraformer for Chinese; SIM = cosine similarity of WavLM-large (UniSpeech wavlm_large_finetune) speaker embeddings between the reference prompt and the synthesized speech (paper: 'mathematically equivalent to SIM-O'); Seed-TTS test-zh/test-en. Same-lab sibling rows (1B and 3.5B) only; the comparator rows are not used (paper Table 1: 'results of other methods are taken from the original paper or, if open-sourced, evaluated by us' - cells not distinguishable).",
    "source_id": "card3-meituan-longcat-longcat-audiodit-3-5b"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-zh-eres2net-sim",
    "score": 0.806,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 3 'Zero-shot TTS performance on SEED-TTS-Eval', test-zh SIM, row 'LongCat-AudioDiT-3.5B [35]', value in parentheses (ERes2Net) = 0.806. Marked '†': 'ERes2Net SIM scores were computed by us using the publicly released models.' The WavLM value outside the parentheses (0.818) is LongCat's own published figure and is not used here.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "seed-tts-en-eres2net-sim",
    "score": 0.771,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 3, test-en SIM, row 'LongCat-AudioDiT-3.5B [35]', value in parentheses (ERes2Net) = 0.771, marked '†' (computed by the Qwen-Audio authors on the publicly released model).",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0924,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-zh subset CER (Paraformer), row 'LongCat-AudioDiT-3.5B' = 9.24. Text: 'For the hard-zh and hard-en subsets in Table 5, we additionally include LongCat-AudioDiT as a bilingual Chinese-English baseline'; open-source comparison systems 'use publicly released open-source models' (Table 4 caption). ERes2Net SIM 72.8% and DNSMOS 3.75 retained as companion metadata.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.085,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, CV3-Eval hard-en subset WER (Whisper-large-v3), row 'LongCat-AudioDiT-3.5B' = 8.50. Publicly released checkpoint run by the Qwen-Audio authors (see hard-zh row).",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.721,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'MiniMax-Speech-2.8-HD', hard-en SIM (%) = 72.1. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "minimax-speech-2-8-hd",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.7,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'MiniMax-Speech-2.8-HD', hard-en DNSMOS = 3.8; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.489,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'ElevenLabs-v3', hard-en SIM (%) = 48.9. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "elevenlabs-v3",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.73,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'ElevenLabs-v3', hard-en DNSMOS = 3.92; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.661,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'Qwen3-TTS-12Hz-1.7B-Base', hard-en SIM (%) = 66.1. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen3-tts-12hz-1-7b-base",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.72,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'Qwen3-TTS-12Hz-1.7B-Base', hard-en DNSMOS = 3.88; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.739,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'LongCat-AudioDiT-3.5B', hard-en SIM (%) = 73.9. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "longcat-audiodit-3-5b",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.71,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'LongCat-AudioDiT-3.5B', hard-en DNSMOS = 3.84; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.76,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'Dots.TTS-2B (SOAR)', hard-en SIM (%) = 76.0. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.68,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'Dots.TTS-2B (SOAR)', hard-en DNSMOS = 3.72; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.67,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'VoxCPM2', hard-en SIM (%) = 67.0. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "voxcpm2",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.6825,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "independent",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'VoxCPM2', hard-en DNSMOS = 3.73; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-hard-en-eres2net-sim",
    "score": 0.766,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5 'WER/CER, ERes2Net speaker similarity (SIM), and DNSMOS on the hard-zh and hard-en subsets of CV3-Eval', row 'Qwen-Audio-3.0-TTS', hard-en SIM (%) = 76.6. Same generations as this source's cv3-eval-hard-en-wer rows (previously kept only as companion metadata); API systems via public APIs, open-source systems from released checkpoints (Table 4 caption). CosyVoice3-1.5B row not catalogued.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "qwen-audio-3-0-tts-plus",
    "bench": "cv3-eval-hard-en-dnsmos",
    "score": 0.76,
    "source": "https://arxiv.org/html/2607.23938v1",
    "by": "lab",
    "date": "2026-07-27",
    "note": "Qwen-Audio-3.0-TTS technical report (Alibaba Token Foundry), arXiv:2607.23938v1, CC BY 4.0. Table 5, row 'Qwen-Audio-3.0-TTS', hard-en DNSMOS = 4.04; stored (DNSMOS-1)/4. Same generations as this source's cv3-eval-hard-en-wer rows.",
    "source_id": "speech-qwen-audio3-tts-report"
  },
  {
    "model": "vibevoice-asr-streaming-7b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0233,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "lab",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'VibeVoice-ASR-Streaming -7B (22 frm)', column 'LS test-clean' WER (%) = 2.33. Released 22-frame (2.9 s) chunk configuration with 4-frame lookahead (the 15-frame row is an unreleased configuration and is not used).",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "vibevoice-asr-streaming-7b",
    "bench": "librispeech-test-other-wer",
    "score": 0.0649,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "lab",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'VibeVoice-ASR-Streaming -7B (22 frm)', column 'LS test-other' WER (%) = 6.49. Released 22-frame (2.9 s) chunk configuration with 4-frame lookahead (the 15-frame row is an unreleased configuration and is not used).",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "vibevoice-asr-streaming-7b",
    "bench": "gigaspeech-wer",
    "score": 0.102,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "lab",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'VibeVoice-ASR-Streaming -7B (22 frm)', column 'GigaSpeech' WER (%) = 10.2. Released 22-frame (2.9 s) chunk configuration with 4-frame lookahead (the 15-frame row is an unreleased configuration and is not used).",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0302,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Nemotron-3.5-ASR 0.6B [17]', column 'LS test-clean' WER (%) = 3.02. Evaluated by Microsoft at att_context_size=[56,13] (1.12 s), 'the released setting closest to our chunk sizes', with the en-US language identifier; not necessarily the model's most accurate setting.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "librispeech-test-other-wer",
    "score": 0.068,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Nemotron-3.5-ASR 0.6B [17]', column 'LS test-other' WER (%) = 6.8. Evaluated by Microsoft at att_context_size=[56,13] (1.12 s), 'the released setting closest to our chunk sizes', with the en-US language identifier; not necessarily the model's most accurate setting.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "nemotron-3-5-asr-streaming-0-6b",
    "bench": "gigaspeech-wer",
    "score": 0.1176,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Nemotron-3.5-ASR 0.6B [17]', column 'GigaSpeech' WER (%) = 11.76. Evaluated by Microsoft at att_context_size=[56,13] (1.12 s), 'the released setting closest to our chunk sizes', with the en-US language identifier; not necessarily the model's most accurate setting.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0188,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Voxtral-Mini-4B-Realtime [15]', column 'LS test-clean' WER (%) = 1.88. Evaluated by Microsoft at transcription_delay_ms=2400 (2.4 s), 'its longest configurable delay'.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "librispeech-test-other-wer",
    "score": 0.041,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Voxtral-Mini-4B-Realtime [15]', column 'LS test-other' WER (%) = 4.1. Evaluated by Microsoft at transcription_delay_ms=2400 (2.4 s), 'its longest configurable delay'.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "voxtral-mini-4b-realtime",
    "bench": "gigaspeech-wer",
    "score": 0.1011,
    "source": "https://arxiv.org/html/2609.02812v2",
    "by": "independent",
    "date": "2026-09-10",
    "note": "VibeVoice-ASR-Streaming Technical Report (Microsoft Research), arXiv:2609.02812v2, CC BY 4.0. Table 3 'Single-speaker recognition', caption note: 'All rows are our own measurements under a single normalization.' Row 'Voxtral-Mini-4B-Realtime [15]', column 'GigaSpeech' WER (%) = 10.11. Evaluated by Microsoft at transcription_delay_ms=2400 (2.4 s), 'its longest configurable delay'.",
    "source_id": "speech-vibevoice-asr-streaming-report-ccby"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "fleurs-en-wer",
    "score": 0.0473,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Fleurs-en', column 'VibeVoice-ASR-7B' = 4.73.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-bitnet",
    "bench": "fleurs-en-wer",
    "score": 0.0521,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Fleurs-en', column 'VibeVoice-ASR-BitNet' = 5.21.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "fleurs-zh-err",
    "score": 0.0792,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Fleurs-zh', column 'VibeVoice-ASR-7B' = 7.92 (CER for Chinese per the report's CER/WER columns; both equal 7.92/8.35).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-bitnet",
    "bench": "fleurs-zh-err",
    "score": 0.0835,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Fleurs-zh', column 'VibeVoice-ASR-BitNet' = 8.35 (CER for Chinese per the report's CER/WER columns; both equal 7.92/8.35).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0217,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Libri-clean', column 'VibeVoice-ASR-7B' = 2.17.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-bitnet",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0241,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Libri-clean', column 'VibeVoice-ASR-BitNet' = 2.41.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "librispeech-test-other-wer",
    "score": 0.0584,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Libri-other', column 'VibeVoice-ASR-7B' = 5.84.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-bitnet",
    "bench": "librispeech-test-other-wer",
    "score": 0.0627,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'Libri-other', column 'VibeVoice-ASR-BitNet' = 6.27.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-hf",
    "bench": "voxpopuli-en-wer",
    "score": 0.0492,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'VoxPopuli', column 'VibeVoice-ASR-7B' = 4.92.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "vibevoice-asr-bitnet",
    "bench": "voxpopuli-en-wer",
    "score": 0.0518,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "lab",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT), 'Accuracy (WER%)' table (same values as arXiv:2607.21075v2 Table 4, WER column). Same-lab columns only: 'VibeVoice-ASR-7B' (FP16; the VibeASR.cpp README labels it 'VibeVoice-ASR-7B (FP16)', i.e. microsoft/VibeVoice-ASR, catalogued as its Transformers release vibevoice-asr-hf) and 'VibeVoice-ASR-BitNet'. Parakeet/Whisper/SenseVoice/FunASR columns not used (no statement that the authors ran them). Row 'VoxPopuli', column 'VibeVoice-ASR-BitNet' = 5.18.",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "gepard-1-0",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.036,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Gepard 1.0 (Ours)', WER = 0.036 (printed as a fraction). The report names only 'Whisper (Radford et al. 2022)' for this table (no model size), hence the transcriber-unstated id.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "gepard-1-0",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.585,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Gepard 1.0 (Ours)', SIM = 0.585 (WavLM speaker embeddings, per the report; reference-cloning mode).",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "gepard-1-0",
    "bench": "seed-tts-en-utmos",
    "score": 0.41,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Gepard 1.0 (Ours)', UTMOS = 2.64; stored (UTMOS-1)/4.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.015,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'VoxCPM2', WER = 0.015 (printed as a fraction). The report names only 'Whisper (Radford et al. 2022)' for this table (no model size), hence the transcriber-unstated id. Only VoxCPM2 checkpoint is openbmb/VoxCPM2.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.867,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'VoxCPM2', SIM = 0.867 (WavLM speaker embeddings, per the report; reference-cloning mode). Only VoxCPM2 checkpoint is openbmb/VoxCPM2.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "voxcpm2",
    "bench": "seed-tts-en-utmos",
    "score": 0.355,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'VoxCPM2', UTMOS = 2.42; stored (UTMOS-1)/4. Only VoxCPM2 checkpoint is openbmb/VoxCPM2.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.016,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Fish-S2', WER = 0.016 (printed as a fraction). The report names only 'Whisper (Radford et al. 2022)' for this table (no model size), hence the transcriber-unstated id. 'Fish-S2' is taken as fishaudio/s2-pro, the only open-weights Fish Audio S2 checkpoint (the paper's cohort is 'open-source voice-cloning systems').",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.789,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Fish-S2', SIM = 0.789 (WavLM speaker embeddings, per the report; reference-cloning mode). 'Fish-S2' is taken as fishaudio/s2-pro, the only open-weights Fish Audio S2 checkpoint (the paper's cohort is 'open-source voice-cloning systems').",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "fish-audio-s2-pro",
    "bench": "seed-tts-en-utmos",
    "score": 0.45,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'Fish-S2', UTMOS = 2.8; stored (UTMOS-1)/4. 'Fish-S2' is taken as fishaudio/s2-pro, the only open-weights Fish Audio S2 checkpoint (the paper's cohort is 'open-source voice-cloning systems').",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-wer-asr-unstated",
    "score": 0.016,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'OmniVoice', WER = 0.016 (printed as a fraction). The report names only 'Whisper (Radford et al. 2022)' for this table (no model size), hence the transcriber-unstated id. OmniVoice = k2-fsa/OmniVoice.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.848,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'OmniVoice', SIM = 0.848 (WavLM speaker embeddings, per the report; reference-cloning mode). OmniVoice = k2-fsa/OmniVoice.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "omnivoice",
    "bench": "seed-tts-en-utmos",
    "score": 0.4075,
    "source": "https://huggingface.co/nineninesix/gepard-1.0/blob/440c3a27797e98673c8e56f0569a86c09ea5de34/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "Gepard 1.0 model card (nineninesix.ai, Apache-2.0) 'Benchmarks': 'Measured on the public Seed-TTS-eval — 1088 paired prompts, identical UUIDs and texts across every model.' Same table as arXiv:2609.04222v1 §6.3 (CC BY 4.0): 'six baselines run with reference cloning ... (seed_tts_eval_en_vc mode) ... our own runs on identical inputs, not a compilation of third-party numbers'; metrics 'WER/CER (Whisper ...), SIM (WavLM ...), UTMOS'. Seed-TTS test-en only (1088 prompts). Row 'OmniVoice', UTMOS = 2.63; stored (UTMOS-1)/4. OmniVoice = k2-fsa/OmniVoice.",
    "source_id": "card3-nineninesix-gepard-1-0"
  },
  {
    "model": "dots-tts-mf",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0129,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). Table 2 'Zero-shot results on Seed-TTS-Eval', row '– MF, NFE=4' (2B), column test-en WER(%) = 1.29. Same settings/scoring as the existing dots.tts (Pretrain) and (SOAR) rows of this table (§3.3.1: Whisper-large-v3 for English, Paraformer for Mandarin, WavLM-SV cosine to the reference prompt). NFE=3/NFE=2 rows not used (non-default settings).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "seed-tts-en-wavlm-sim",
    "score": 0.762,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). Table 2 'Zero-shot results on Seed-TTS-Eval', row '– MF, NFE=4' (2B), column test-en SIM = 76.2. Same settings/scoring as the existing dots.tts (Pretrain) and (SOAR) rows of this table (§3.3.1: Whisper-large-v3 for English, Paraformer for Mandarin, WavLM-SV cosine to the reference prompt). NFE=3/NFE=2 rows not used (non-default settings).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0094,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). Table 2 'Zero-shot results on Seed-TTS-Eval', row '– MF, NFE=4' (2B), column test-zh WER (CER for Chinese) = 0.94. Same settings/scoring as the existing dots.tts (Pretrain) and (SOAR) rows of this table (§3.3.1: Whisper-large-v3 for English, Paraformer for Mandarin, WavLM-SV cosine to the reference prompt). NFE=3/NFE=2 rows not used (non-default settings).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "seed-tts-zh-wavlm-sim",
    "score": 0.8,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). Table 2 'Zero-shot results on Seed-TTS-Eval', row '– MF, NFE=4' (2B), column test-zh SIM = 80.0. Same settings/scoring as the existing dots.tts (Pretrain) and (SOAR) rows of this table (§3.3.1: Whisper-large-v3 for English, Paraformer for Mandarin, WavLM-SV cosine to the reference prompt). NFE=3/NFE=2 rows not used (non-default settings).",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "cv3-eval-en-wer",
    "score": 0.0524,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (Pretrain)', column Monolingual en W(%) = 5.24. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0969,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (Pretrain)', column Monolingual hard-zh W(%) = 9.69. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-base",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0599,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (Pretrain)', column Monolingual hard-en W(%) = 5.99. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-en-wer",
    "score": 0.045,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (SOAR)', column Monolingual en W(%) = 4.5. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.0922,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (SOAR)', column Monolingual hard-zh W(%) = 9.22. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-soar",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0449,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (SOAR)', column Monolingual hard-en W(%) = 4.49. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "cv3-eval-en-wer",
    "score": 0.0405,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (MF, NFE=4)', column Monolingual en W(%) = 4.05. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "cv3-eval-hard-zh-cer",
    "score": 0.091,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (MF, NFE=4)', column Monolingual hard-zh W(%) = 9.1. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "dots-tts-mf",
    "bench": "cv3-eval-hard-en-wer",
    "score": 0.0437,
    "source": "https://arxiv.org/html/2606.07080v2",
    "by": "lab",
    "date": "2026-08-10",
    "note": "dots.tts Technical Report, arXiv:2606.07080v2 (CC BY 4.0, dots.tts Team). §3.3.5 Table 4 'Results on CV3-Eval. W = WER, S = SIM.', row 'dots.tts (MF, NFE=4)', column Monolingual hard-en W(%) = 4.37. Same-lab rows only; baseline rows (CosyVoice 2/3, Fish-Audio S2, VoxCPM 2) not used because §3.3.1 says 'All baseline numbers reported in this section are taken either from the original papers or from the official default inference configuration', without marking which.",
    "source_id": "speech-dots-tts-report"
  },
  {
    "model": "lemas-tts-multilingual-grl",
    "bench": "lemas-eval-9lang-wer",
    "score": 0.057178,
    "source": "https://arxiv.org/html/2601.04233v1",
    "by": "lab",
    "date": "2026-01-04",
    "note": "LEMAS report, arXiv:2601.04233v1, CC BY-SA 4.0. Table 4 'Objective evaluation of TTS models on multilingual test sets', rows 'LEMAS-TTS w/o prosody'. DERIVED unweighted mean of the nine languages used by this benchmark (de,en,es,fr,id,it,pt,ru,zh; vi excluded as for the OpenAudio-S1-Mini row) = 5.7178%. The printed 'Avg' (8.06) matches neither the 9-language mean nor the 10-language mean incl. vi (7.360), so it is not used. Same evaluation set and scoring as the existing S1-Mini row (500 utterances per language; Paraformer-zh / Whisper-large-v3). Authors' own model: by lab. WavLM SIM avg 0.547 kept as companion.",
    "source_id": "speech-lemas-report-bysa"
  },
  {
    "model": "lemas-tts-multilingual-prosody",
    "bench": "lemas-eval-9lang-wer",
    "score": 0.048222,
    "source": "https://arxiv.org/html/2601.04233v1",
    "by": "lab",
    "date": "2026-01-04",
    "note": "LEMAS report, arXiv:2601.04233v1, CC BY-SA 4.0. Table 4 'Objective evaluation of TTS models on multilingual test sets', rows 'LEMAS-TTS w/ prosody'. DERIVED unweighted mean of the nine languages used by this benchmark (de,en,es,fr,id,it,pt,ru,zh; vi excluded as for the OpenAudio-S1-Mini row) = 4.8222%. The printed 'Avg' (6.39) matches neither the 9-language mean nor the 10-language mean incl. vi (5.878), so it is not used. Same evaluation set and scoring as the existing S1-Mini row (500 utterances per language; Paraformer-zh / Whisper-large-v3). Authors' own model: by lab. WavLM SIM avg 0.539 kept as companion.",
    "source_id": "speech-lemas-report-bysa"
  },
  {
    "model": "muse-voice-transcribe-1-0",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0073,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Meta | muse-voice-transcribe-1.0': Transcripts 99.9%, Perfect 89.5%, WER Mean 0.80%, Pooled WER 0.73%. services.py create_meta(): MetaSTTService model='muse-voice-transcribe-1.0', language EN (streaming WebSocket). Added in commit 62a4ff7 ('Add Meta benchmark results to README and plots', 2026-09-11). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "assemblyai-universal-3-6-pro-realtime",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.008,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'AssemblyAI | universal-3-6-pro': Transcripts 99.9%, Perfect 88.7%, WER Mean 0.89%, Pooled WER 0.80%. services.py create_assemblyai_universal_3_6_pro(): AssemblyAISTTService (streaming) model='universal-3-6-pro', min_turn_silence=50, vad_threshold=0.2, language_codes=[EN]. Results added in commit c6b7d83 (2026-09-17); the service config came from AssemblyAI staff (PR #39, dlange-aai) but Pipecat ran and published it. Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "assemblyai-universal-streaming-english",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0242,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'AssemblyAI | universal-streaming-english': Transcripts 99.8%, Perfect 66.8%, WER Mean 2.81%, Pooled WER 2.42%. services.py create_assemblyai(): AssemblyAISTTService model='universal-streaming-english', end_of_turn_confidence_threshold=1.0, max_turn_silence=2000. The code comment says universal-3-5-pro supersedes this entry; the row stays published. Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "cartesia-ink-2",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0102,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Cartesia | ink-2': Transcripts 100.0%, Perfect 83.6%, WER Mean 1.14%, Pooled WER 1.02%. services.py create_cartesia_ink2(): CartesiaSTTService model='ink-2', language EN. Data added 2026-05-28 (commit 5c1c6f6, 'Update settings, add Cartesia ink-2 data'), i.e. the May 2026 English-only Ink-2, before the 2026-09-17 multilingual update of the alias. Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "cartesia-ink-whisper",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0378,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Cartesia | ink-whisper': Transcripts 99.9%, Perfect 61.4%, WER Mean 3.32%, Pooled WER 3.78%. services.py create_cartesia(): CartesiaSTTService model='ink-whisper', language EN (in the repository's initial commit 29b4f95, 2026-02-09). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "elevenlabs-scribe-v2-realtime",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0313,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'ElevenLabs | scribe_v2_realtime': Transcripts 99.7%, Perfect 80.3%, WER Mean 3.05%, Pooled WER 3.13%. services.py create_elevenlabs(): ElevenLabsRealtimeSTTService model='scribe_v2_realtime', language EN (present since the initial commit 29b4f95, 2026-02-09). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "gemini-3-5-transcribe-live",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0197,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Google | gemini-3.5-transcribe-live': Transcripts 99.9%, Perfect 78.3%, WER Mean 1.93%, Pooled WER 1.97%. services.py create_google_gemini_3_5_transcribe_live(): GeminiSTTService model='gemini-3.5-transcribe-live', languages [EN_US]. Added in commit 55824d5 ('Add Gemini STT service and its benchmark results', 2026-08-27). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "gpt-realtime-whisper",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.023,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'OpenAI | gpt-realtime-whisper': Transcripts 100.0%, Perfect 73.3%, WER Mean 2.41%, Pooled WER 2.30%. services.py create_openai_gpt_realtime_whisper(): OpenAIRealtimeSTTService model='gpt-realtime-whisper', language EN. Added in commit 50df24f ('Add gpt-realtime-whisper benchmark results', 2026-08-17). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "soniox-v5-realtime",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0109,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Soniox | stt-rt-v5': Transcripts 99.8%, Perfect 83.6%, WER Mean 1.11%, Pooled WER 1.09%. services.py create_soniox_stt_rt_v5(): SonioxSTTService model='stt-rt-v5', language_hints [EN], language_hints_strict=True, vad_force_turn_endpoint=True. Added in commit 13798eb ('Add soniox stt-rt-v5 service and results', 2026-06-22). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "soniox-v4-realtime",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0113,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Soniox | stt-rt-v4': Transcripts 99.8%, Perfect 83.2%, WER Mean 1.04%, Pooled WER 1.13%. services.py create_soniox(): SonioxSTTService model='stt-rt-v4' (in the initial commit 29b4f95, 2026-02-09, after the 2026-02-05 release and before the 2026-06-30 retirement; TTFS figures unchanged at 9e24957 and 94f5672, i.e. the transcripts were not re-run after v4 was routed to v5). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "speechmatics-linden-1",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0096,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Speechmatics | linden-1': Transcripts 99.5%, Perfect 83.9%, WER Mean 1.11%, Pooled WER 0.96%. services.py create_speechmatics_agent_stt(): SpeechmaticsSTTService base_url wss://us.rt.speechmatics.com/v2/agent, language EN, turn_detection_mode EXTERNAL; code comment 'Speechmatics Agent STT (/v2/agent, model linden-1)'. Added in commit ecb476a ('Add Speechmatics Agent STT service and benchmark results', 2026-09-15). Speechmatics' own article quotes 1.05% from the earlier judge; the current README value is used. Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "smallest-pulse",
    "bench": "pipecat-stt-semantic-wer-pooled",
    "score": 0.0194,
    "source": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Row 'Smallest AI | pulse': Transcripts 100.0%, Perfect 74.0%, WER Mean 1.85%, Pooled WER 1.94%. services.py create_smallest(): SmallestSTTService model='pulse', language EN (streaming). Smallest added in commit b79bc7d (2026-05-14) / 5a677ef (2026-05-16). Scored by the current Claude Sonnet 5.5 judge (not a † row); re-scored in commit f78a7a2 (2026-10-01). Service config: https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/src/stt_benchmark/services.py.",
    "source_id": "speech-stt-pipecat-bsd2"
  },
  {
    "model": "parakeet-ultra",
    "bench": "granite5-report-oasr-public7-wer",
    "score": 0.058,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'Open ASR Leaderboard' (LibriSpeech test-clean, LibriSpeech test-other, AMI, Earnings-22, GigaSpeech, SPGISpeech, VoxPopuli), row '**average**', column 'parakeet-ultra' = 5.8% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.'",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-ultra",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.0955,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'FLEURS' (FLEURS test split, all 25 languages the model supports: bg, hr, cs, da, nl, en, et, fi, fr, de, el, hu, it, lv, lt, mt, pl, pt, ro, ru, sk, sl, es, sv, uk), row '**average**', column 'parakeet-ultra' = 9.55% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.'",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-ultra",
    "bench": "tedlium-longform-wer",
    "score": 0.0194,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'Long-form' ('Eleven complete TED-LIUM 3 talks of 10-20 minutes each'), row 'TED-LIUM 3, 11 full talks of 10-20 minutes', column 'parakeet-ultra' = 1.94% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.' Photon segments each talk at pauses found by the model's own VAD head into segments of at most 30 s; the original runs NeMo's long-audio path.",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "granite5-report-oasr-public7-wer",
    "score": 0.0626,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "independent",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'Open ASR Leaderboard' (LibriSpeech test-clean, LibriSpeech test-other, AMI, Earnings-22, GigaSpeech, SPGISpeech, VoxPopuli), row '**average**', column 'parakeet-tdt-0.6b-v3' = 6.26% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.' NVIDIA's checkpoint run by Moondream in NeMo bf16; the parakeet-redux card prints the identical column (same measurement), so it is recorded once.",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.1162,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "independent",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'FLEURS' (FLEURS test split, all 25 languages the model supports: bg, hr, cs, da, nl, en, et, fi, fr, de, el, hu, it, lv, lt, mt, pl, pt, ro, ru, sk, sl, es, sv, uk), row '**average**', column 'parakeet-tdt-0.6b-v3' = 11.62% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.' NVIDIA's checkpoint run by Moondream in NeMo bf16; identical in the parakeet-redux card, recorded once.",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "tedlium-longform-wer",
    "score": 0.0271,
    "source": "https://huggingface.co/moondream/parakeet-ultra/blob/73175eb7aeb0d82f1e2a6b53b3aabc10a90bcd0b/README.md",
    "by": "independent",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-ultra @73175eb), Benchmarks section, table 'Long-form' ('Eleven complete TED-LIUM 3 talks of 10-20 minutes each'), row 'TED-LIUM 3, 11 full talks of 10-20 minutes', column 'parakeet-tdt-0.6b-v3' = 2.71% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-ultra runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.' NeMo long-audio path; identical in the parakeet-redux card, recorded once.",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-redux",
    "bench": "granite5-report-oasr-public7-wer",
    "score": 0.0655,
    "source": "https://huggingface.co/moondream/parakeet-redux/blob/2bf128600aac4b16946f7ed8372e56117fe5e23b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-redux @2bf1286), Benchmarks section, table 'Open ASR Leaderboard' (LibriSpeech test-clean, LibriSpeech test-other, AMI, Earnings-22, GigaSpeech, SPGISpeech, VoxPopuli), row '**average**', column 'parakeet-redux' = 6.55% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-redux runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.'",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-redux",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.1056,
    "source": "https://huggingface.co/moondream/parakeet-redux/blob/2bf128600aac4b16946f7ed8372e56117fe5e23b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-redux @2bf1286), Benchmarks section, table 'FLEURS' (FLEURS test split, all 25 languages the model supports: bg, hr, cs, da, nl, en, et, fi, fr, de, el, hu, it, lv, lt, mt, pl, pt, ro, ru, sk, sl, es, sv, uk), row '**average**', column 'parakeet-redux' = 10.56% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-redux runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.'",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "parakeet-redux",
    "bench": "tedlium-longform-wer",
    "score": 0.0251,
    "source": "https://huggingface.co/moondream/parakeet-redux/blob/2bf128600aac4b16946f7ed8372e56117fe5e23b/README.md",
    "by": "lab",
    "date": "2026-09-23",
    "note": "Moondream card (parakeet-redux @2bf1286), Benchmarks section, table 'Long-form' ('Eleven complete TED-LIUM 3 talks of 10-20 minutes each'), row 'TED-LIUM 3, 11 full talks of 10-20 minutes', column 'parakeet-redux' = 2.51% WER. Card: 'Both models are scored on the same files with the Open ASR Leaderboard's own pipeline as of September 2026 (its normalizers and compound-merging alignment, with the FLEURS references prepared as the leaderboard's text column is). parakeet-redux runs in Photon on an NVIDIA GPU, the original in NeMo in bf16.' Photon VAD-head segmentation (segments of at most 30 s).",
    "source_id": "card3-moondream-parakeet-cards"
  },
  {
    "model": "gigaam-multilingual-ctc",
    "bench": "fleurs-en-wer",
    "score": 0.122,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "lab",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | FLEURS', column 'GigaAM Multilingual' = 12.2% WER. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Same-lab sibling columns (GigaAM Multilingual = ctc 220M, GigaAM Multilingual Large = large_ctc 600M). Digit-containing references excluded, so not the full FLEURS en_us test.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "gigaam-multilingual-large-ctc",
    "bench": "fleurs-en-wer",
    "score": 0.094,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "lab",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | FLEURS', column 'GigaAM Multilingual Large' = 9.4% WER. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Same-lab sibling columns (GigaAM Multilingual = ctc 220M, GigaAM Multilingual Large = large_ctc 600M). Digit-containing references excluded, so not the full FLEURS en_us test.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "gigaam-multilingual-ctc",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.26,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "lab",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | CV', column 'GigaAM Multilingual' = 26.0% WER. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Same-lab sibling columns (GigaAM Multilingual = ctc 220M, GigaAM Multilingual Large = large_ctc 600M). Common Voice version not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "gigaam-multilingual-large-ctc",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.215,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "lab",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | CV', column 'GigaAM Multilingual Large' = 21.5% WER. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Same-lab sibling columns (GigaAM Multilingual = ctc 220M, GigaAM Multilingual Large = large_ctc 600M). Common Voice version not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "livebench-20240831",
    "score": 0.4957,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-flash-002'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 49.57. Category means: math 47.2, coding 41.9, reasoning 50.0, language 29.5, instruction_following 84.5, data_analysis 44.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: stable API version gemini-1.5-flash-002; answers generated 2024-09-24..25.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "livebench-20240831",
    "score": 0.5494,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-pro-002'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 54.94. Category means: math 57.4, coding 48.8, reasoning 46.0, language 47.4, instruction_following 77.7, data_analysis 52.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: stable API version gemini-1.5-pro-002; answers generated 2024-09-24.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "llama-3-1-nemotron-70b",
    "bench": "livebench-20240831",
    "score": 0.4311,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'llama-3.1-nemotron-70b-instruct'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 43.11. Category means: math 37.6, coding 32.7, reasoning 48.0, language 31.8, instruction_following 69.6, data_analysis 39.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: NVIDIA Llama-3.1-Nemotron-70B-Instruct; answers generated 2024-10-16..17.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "o1-mini",
    "bench": "livebench-20240831",
    "score": 0.5909,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'o1-mini-2024-09-12'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 59.09. Category means: math 59.2, coding 48.1, reasoning 77.3, language 45.7, instruction_following 70.2, data_analysis 54.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: dated API snapshot o1-mini-2024-09-12 (the only o1-mini snapshot); default reasoning (o1-mini had no reasoning-effort parameter at the time); answers generated 2024-09-12..13.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "o1-preview",
    "bench": "livebench-20240831",
    "score": 0.6602,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'o1-preview-2024-09-12'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 66.02. Category means: math 62.9, coding 50.8, reasoning 68.0, language 72.7, instruction_following 77.7, data_analysis 64.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: dated API snapshot o1-preview-2024-09-12 (the only o1-preview snapshot); default reasoning (no effort parameter); answers generated 2024-09-12..13.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "livebench-20240831",
    "score": 0.5289,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2.5-72b-instruct'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; release/removal dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 52.89. Category means: math 52.4, coding 56.6, reasoning 46.0, language 38.1, instruction_following 75.5, data_analysis 48.8. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: Qwen/Qwen2.5-72B-Instruct; answers generated 2024-09-20.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "lmarena-text",
    "score": 1386.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "text_style_control, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'mimo-v2-flash (thinking)'; 11,143 votes; published 95% interval 1380.2–1392.5. Thinking mode, the highest effort Arena lists for MiMo-V2-Flash (the 'mimo-v2-flash (non-thinking)' entry, 1391.6, is the same weights in non-thinking mode and is not duplicated). Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "lmarena-webdev",
    "score": 1292.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/webdev/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "webdev, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'mimo-v2-flash (thinking)'; 1,687 votes; published 95% interval 1276.0–1308.9. Thinking mode, the highest effort Arena lists for MiMo-V2-Flash (the non-thinking entry, 1330.2, is not duplicated). Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "llama-13b",
    "bench": "lmarena-text",
    "score": 975,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "text_style_control, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'llama-13b'; 2,391 votes; published 95% interval 959.4–990.7. Arena's leaderboard table (arena-leaderboard Space @6f2eeeb, leaderboard_table_20250804.csv) labels this key 'LLaMA-13B' (Meta, Non-commercial) linking arXiv 2302.13971, i.e. the original LLaMA v1 13B. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "dbrx-instruct",
    "bench": "lmarena-text",
    "score": 1195.5,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "text_style_control, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'dbrx-instruct-preview'; 32,191 votes; published 95% interval 1189.4–1201.6. Arena's leaderboard table (arena-leaderboard Space @6f2eeeb, leaderboard_table_20250804.csv) labels this key 'DBRX-Instruct-Preview' (Databricks, 'DBRX LICENSE') linking Databricks' 'Introducing DBRX' post; DBRX Instruct is the only DBRX instruct checkpoint Databricks released. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "lmarena-webdev",
    "score": 1586.9,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/webdev/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "webdev, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'claude-haiku-5.5-high'; 978 votes; published 95% interval 1567.3–1606.5. Only Claude Haiku 5.5 entry in this snapshot (effort 'high'); first published 2026-10-08, so only 978 votes and a wide interval. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-6-pro",
    "bench": "arena-agent-ips-202610",
    "score": 0.03524172926571822,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/agent/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "agent latest overall (leaderboard_publish_date 2026-10-08); exact entry 'MiMo V2.6 Pro'; highest available reasoning effort for this model in this task (the only entry for it). Aggregate IPS net improvement, not Elo; 95% CI [0.016064735465072696, 0.054418723066363736]; 11,376 sessions, 988,415 observations. Same aggregate as the existing October 2 rows (October 1, 2026 signal definitions; the 50 entries common to both snapshots moved by a median 0.003 IPS). Lower effort and the five constituent signals are not duplicated. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mimo-v2-6-flash",
    "bench": "arena-agent-ips-202610",
    "score": 0.0035286050197275384,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/agent/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "agent latest overall (leaderboard_publish_date 2026-10-08); exact entry 'MiMo V2.6 Flash'; highest available reasoning effort for this model in this task (the only entry for it). Aggregate IPS net improvement, not Elo; 95% CI [-0.011206337254353358, 0.018263547293808434]; 16,864 sessions, 1,796,644 observations. Same aggregate as the existing October 2 rows (October 1, 2026 signal definitions; the 50 entries common to both snapshots moved by a median 0.003 IPS). Lower effort and the five constituent signals are not duplicated. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mistral-large-4",
    "bench": "lmarena-text",
    "score": 1429.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "text_style_control, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'mistral-large-4'; 4,179 votes; published 95% interval 1420.0–1438.5. Only Mistral Large 4 entry; first published 2026-10-08 (single publish so far). Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mistral-large-4",
    "bench": "lmarena-webdev",
    "score": 1541.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/webdev/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "webdev, category overall, split latest (leaderboard_publish_date 2026-10-08); exact entry 'mistral-large-4'; 1,478 votes; published 95% interval 1524.9–1557.8. Only Mistral Large 4 entry; first published 2026-10-08 (single publish so far). Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "mistral-large-4",
    "bench": "arena-agent-ips-202610",
    "score": -0.06560145031424472,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/agent/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "agent latest overall (leaderboard_publish_date 2026-10-08); exact entry 'mistral-large-4'; highest available reasoning effort for this model in this task (the only entry for it). Aggregate IPS net improvement, not Elo; 95% CI [-0.09508835660601413, -0.03611454402247531]; 5,078 sessions, 271,865 observations. Same aggregate as the existing October 2 rows (October 1, 2026 signal definitions; the 50 entries common to both snapshots moved by a median 0.003 IPS). Lower effort and the five constituent signals are not duplicated. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "gpt-4o-search-preview",
    "bench": "lmarena-search",
    "score": 1081,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/search_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-08-24",
    "note": "search_style_control, category overall, split latest (leaderboard_publish_date 2026-08-24); exact entry 'api-gpt-4o-search'; 3,411 votes; published 95% interval 1069.6–1092.4. Search system entry (organization openai). Mapped to OpenAI's GPT-4o Search Preview model; Arena's open Search Arena dataset (lmarena-ai/search-arena-24k) names the same system 'gpt-4o-search-preview' (default and '-high' search_context_size settings), and this leaderboard does not say which search_context_size the current entry uses. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "nvila-15b",
    "bench": "lmarena-vision",
    "score": 985.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/vision_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-02",
    "note": "vision_style_control, category overall, split latest (leaderboard_publish_date 2026-10-02); exact entry 'nvila-internal-15b-v1'; 1,077 votes; published 95% interval 965.3–1005.5. Arena's leaderboard table (arena-leaderboard Space @6f2eeeb, leaderboard_table_20250804.csv) labels this key 'NVILA-15B' (NVIDIA) linking huggingface.co/Efficient-Large-Model/NVILA-15B. Small sample: 1,077 votes. Pinned to dataset revision 088c6079 (main on 2026-10-10; commit 'Update agent_tool_hallucination for 2026-10-08').",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "pai-bench-g-overall",
    "score": 0.814,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'Cosmos-Predict2.5-2B [4]', column 'Overall' = 81.4 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12. Table 12 Version 'base/post-trained'; NVIDIA model, independent evaluators.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "pai-bench-g-human-elo",
    "score": 1008.1,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'Cosmos-Predict2.5-2B [4]', column 'ELO Scores / Overall' = 1008.1. Arena-style human pairwise study on identical captions. Table 12 Version 'base/post-trained'; NVIDIA model, independent evaluators.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "magi-1-24b",
    "bench": "pai-bench-g-overall",
    "score": 0.794,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'MAGI-1-24B [75]', column 'Overall' = 79.4 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "magi-1-24b",
    "bench": "pai-bench-g-human-elo",
    "score": 988.9,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'MAGI-1-24B [75]', column 'ELO Scores / Overall' = 988.9. Arena-style human pairwise study on identical captions.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "magi-1-4-5b",
    "bench": "pai-bench-g-overall",
    "score": 0.775,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'MAGI-1-4.5B [75]', column 'Overall' = 77.5 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "cosmos-predict2-2b-video2world",
    "bench": "pai-bench-g-overall",
    "score": 0.797,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'Cosmos-Predict2-2B [4]', column 'Overall' = 79.7 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "cosmos-predict2-14b-video2world",
    "bench": "pai-bench-g-overall",
    "score": 0.8,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'Cosmos-Predict2-14B [4]', column 'Overall' = 80.0 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "wan-2-1-i2v-14b-720p",
    "bench": "pai-bench-g-overall",
    "score": 0.808,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'Wan2.1-I2V-14B [79]', column 'Overall' = 80.8 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12. Table 12 Version 'Wan2.1-I2V-14B-720P'.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "wan-2-1-i2v-14b-720p",
    "bench": "pai-bench-g-human-elo",
    "score": 976,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'Wan2.1-I2V-14B [79]', column 'ELO Scores / Overall' = 976.0. Arena-style human pairwise study on identical captions. Table 12 Version 'Wan2.1-I2V-14B-720P'.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "dynamicrafter-1024",
    "bench": "pai-bench-g-overall",
    "score": 0.683,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Table 3 'Evaluation results of 15 VGMs on PAI-Bench-G', row 'DynamicCrafter [93]', column 'Overall' = 68.3 (stored /100): mean of Domain Score and Quality Score; image-conditioned generation. Checkpoint per Table 12. Table 12 Version 'DynamiCrafter_1024'.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "dynamicrafter-1024",
    "bench": "pai-bench-g-human-elo",
    "score": 734.9,
    "source": "https://arxiv.org/html/2512.01989v1",
    "by": "independent",
    "date": "2025-12-01",
    "note": "PAI-Bench (arXiv 2512.01989v1, Georgia Tech / CMU, CC BY 4.0) Appendix Table 8 'Human-study (ELO) vs. MLLM evaluation on PAI-Bench-G', row 'DynamicCrafter [93]', column 'ELO Scores / Overall' = 734.9. Arena-style human pairwise study on identical captions. Table 12 Version 'DynamiCrafter_1024'.",
    "source_id": "paper-2512.01989v1-paibench"
  },
  {
    "model": "cosmos-predict2-5-14b",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.821,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos-Predict2.5-14B' (Open-sourced), column 'Cosmos HUE / Text-to-Video' = 82.1 -> 82.1/100. Same human-annotated atomic Yes/No verification run as the imported Cosmos3/Wan/HunyuanVideo/Veo/Seedance cells; NVIDIA evaluating its own earlier model, so by: lab.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos-predict2-5-14b",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.83,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos-Predict2.5-14B' (Open-sourced), column 'Cosmos HUE / Image-to-Video' = 83.0 -> 83.0/100. Same human-annotated atomic Yes/No verification run as the imported Cosmos3/Wan/HunyuanVideo/Veo/Seedance cells; NVIDIA evaluating its own earlier model, so by: lab.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "cosmos-hue-t2v-human",
    "score": 0.818,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos-Predict2.5-2B' (Open-sourced), column 'Cosmos HUE / Text-to-Video' = 81.8 -> 81.8/100. Same human-annotated atomic Yes/No verification run as the imported Cosmos3/Wan/HunyuanVideo/Veo/Seedance cells; NVIDIA evaluating its own earlier model, so by: lab.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "cosmos-hue-i2v-human",
    "score": 0.826,
    "source": "https://arxiv.org/html/2606.02800v4#S6.T14",
    "by": "lab",
    "date": "2026-06-23",
    "note": "Cosmos 3 report (NVIDIA, arXiv 2606.02800v4, CC BY 4.0) Table 14, row 'Cosmos-Predict2.5-2B' (Open-sourced), column 'Cosmos HUE / Image-to-Video' = 82.6 -> 82.6/100. Same human-annotated atomic Yes/No verification run as the imported Cosmos3/Wan/HunyuanVideo/Veo/Seedance cells; NVIDIA evaluating its own earlier model, so by: lab.",
    "source_id": "visual-cosmos3-report"
  },
  {
    "model": "hy-omniweaving",
    "bench": "phyground-human-overall",
    "score": 0.62,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'OmniWeaving'; Table 2 PhyGround human Overall 3.10 on 1-5 -> 3.10/5 (352-annotator post-QC pool, TI2V). Appendix generation-settings table names the checkpoint (OmniWeaving, 848x480, 81 frames, CFG 6, 50 steps).",
    "source_id": "visual-phyground"
  },
  {
    "model": "cosmos-predict2-5-14b",
    "bench": "phyground-human-overall",
    "score": 0.582,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'Cosmos-14B'; Table 2 PhyGround human Overall 2.91 on 1-5 -> 2.91/5 (352-annotator post-QC pool, TI2V). Appendix generation-settings table names the checkpoint (Cosmos-Predict2.5-14B).",
    "source_id": "visual-phyground"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "phyground-human-overall",
    "score": 0.516,
    "source": "https://arxiv.org/html/2605.10806v1#S4.T2",
    "by": "independent",
    "date": "2026-05-11",
    "note": "'Cosmos-2B'; Table 2 PhyGround human Overall 2.58 on 1-5 -> 2.58/5 (352-annotator post-QC pool, TI2V). Appendix generation-settings table names the checkpoint (Cosmos-Predict2.5-2B).",
    "source_id": "visual-phyground"
  },
  {
    "model": "wan-2-1-vace-14b",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1506.7,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'Wan-VACE 14B'; Table 2 human Bradley-Terry Elo 1506.7 (95% CI [1429.9, 1590.8], 52 games); open weights run locally (Table 1: edit/transfer interface class). Row previously withheld only because the model was not catalogued.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "hy-omniweaving",
    "bench": "wrop-object-permanence-human-elo",
    "score": 1268.5,
    "source": "https://arxiv.org/html/2609.28654v1#S5.T2",
    "by": "independent",
    "date": "2026-09-23",
    "note": "'HY-OmniWeaving'; Table 2 human Bradley-Terry Elo 1268.5 (95% CI [1159.4, 1344.2], 50 games); open weights run locally (Table 1: edit/transfer interface class). Row previously withheld only because the model was not catalogued.",
    "source_id": "visual-wrop-object-permanence"
  },
  {
    "model": "hy-omniweaving",
    "bench": "videoargus-t2v",
    "score": 0.84,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'OmniWeaving | 8.3B (T2V)' in the T2V leaderboard; Table 2 VideoArgus score 8.40/10 -> /10; open-source model evaluated by the VideoArgus authors (Sec. 5: 'We also evaluate a broad set of open-source models').",
    "source_id": "visual-videoargus"
  },
  {
    "model": "hy-omniweaving",
    "bench": "videoargus-ti2v",
    "score": 0.799,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'OmniWeaving | 8.3B (TI2V)' in the TI2V leaderboard; Table 2 VideoArgus score 7.99/10 -> /10; open-source model evaluated by the VideoArgus authors (Sec. 5: 'We also evaluate a broad set of open-source models').",
    "source_id": "visual-videoargus"
  },
  {
    "model": "stepvideo-t2v",
    "bench": "videoargus-t2v",
    "score": 0.777,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Step-Video | 29B (T2V)' in the T2V leaderboard; Table 2 VideoArgus score 7.77/10 -> /10; open-source model evaluated by the VideoArgus authors (Sec. 5: 'We also evaluate a broad set of open-source models').",
    "source_id": "visual-videoargus"
  },
  {
    "model": "stepvideo-ti2v",
    "bench": "videoargus-ti2v",
    "score": 0.711,
    "source": "https://arxiv.org/html/2608.05485v1#S3.T2",
    "by": "independent",
    "date": "2026-08-06",
    "note": "'Step-Video | 29B (TI2V)' in the TI2V leaderboard; Table 2 VideoArgus score 7.11/10 -> /10; open-source model evaluated by the VideoArgus authors (Sec. 5: 'We also evaluate a broad set of open-source models').",
    "source_id": "visual-videoargus"
  },
  {
    "model": "univideo",
    "bench": "worldreasonbench-human-elo",
    "score": 665,
    "source": "https://arxiv.org/html/2605.10434v1#S4.T4",
    "by": "independent",
    "date": "2026-05-11",
    "note": "UniVideo; Table 4 (S4.T4) Human Elo 665 (human rank 10 of 11); expert Bradley-Terry/Davidson fit over WorldRewardBench preference pairs; TI2V (initial image + instruction); open-source generator run by the authors.",
    "source_id": "video-worldreasonbench"
  },
  {
    "model": "minimax-h3-max",
    "bench": "rapidata-relative-camera-i2v",
    "score": 1271.6,
    "source": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md#overall-ranking-elo-on-the-alignment-leaderboard",
    "by": "independent",
    "date": "2026-09-29",
    "note": "MiniMax H3 Max (MiniMax, commercial image-to-video); Alignment-leaderboard ELO 1271.6 (rank 4 of 15); public API endpoint, default settings, I2V.",
    "source_id": "visual-rapid-relative-camera"
  },
  {
    "model": "minimax-h3-max",
    "bench": "rapidata-i2v-realism-overall",
    "score": 1194.15,
    "source": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md#overall-ranking",
    "by": "independent",
    "date": "2026-10-01",
    "note": "minimax-h3-max; Overall ranking ELO 1194.15 (rank 9); Rapidata I2V realism & visual quality; listed separately from minimax-h3 (1324.98).",
    "source_id": "visual-rapid-i2v-realism"
  },
  {
    "model": "magicbrush",
    "bench": "edival-o-turn3",
    "score": 0.1941,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'†MagicBrush' († = open-sourced model); Table 3 EdiVal-O at turn 3 19.41 -> /100 (same table and column as the imported rows); weights vinesmsuic/magicbrush-jul7 per Appendix.",
    "source_id": "visual-edival"
  },
  {
    "model": "anyedit",
    "bench": "edival-o-turn3",
    "score": 0.225,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'†AnyEdit' († = open-sourced model); Table 3 EdiVal-O at turn 3 22.50 -> /100 (same table and column as the imported rows); AnySD repository @9e7d36e, edit_type=general per Appendix.",
    "source_id": "visual-edival"
  },
  {
    "model": "ultraedit-sd3",
    "bench": "edival-o-turn3",
    "score": 0.2289,
    "source": "https://arxiv.org/html/2509.13399v3#S3.T3",
    "by": "independent",
    "date": "2026-03-17",
    "note": "'†UltraEdit' († = open-sourced model); Table 3 EdiVal-O at turn 3 22.89 -> /100 (same table and column as the imported rows); UltraEdit settings with a black mask per Appendix.",
    "source_id": "visual-edival"
  },
  {
    "model": "anyedit",
    "bench": "edit-compass-en-overall",
    "score": 0.262,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "'AnyEdit Yu et al. (2025)'; S4.T3 English-instruction Overall (AVG) 1.31 on the 1-5 Gemini-3.1-Pro judge scale, stored as 1.31/5. Open model; default settings not further specified.",
    "source_id": "visual-edit-compass"
  },
  {
    "model": "nextstep-1-large-edit",
    "bench": "edit-compass-en-overall",
    "score": 0.29,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "'Nextstep-V1 Han et al. (2025a)'; S4.T3 English-instruction Overall (AVG) 1.45 on the 1-5 Gemini-3.1-Pro judge scale, stored as 1.45/5. Open model; default settings not further specified.",
    "source_id": "visual-edit-compass"
  },
  {
    "model": "deepgen-1-0",
    "bench": "edit-compass-en-overall",
    "score": 0.382,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "'DeepGen 1.0 Wang et al. (2026a)'; S4.T3 English-instruction Overall (AVG) 1.91 on the 1-5 Gemini-3.1-Pro judge scale, stored as 1.91/5. Open model; default settings not further specified.",
    "source_id": "visual-edit-compass"
  },
  {
    "model": "unipic3",
    "bench": "edit-compass-en-overall",
    "score": 0.47,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "'Unipic3 Wei et al. (2026)'; S4.T3 English-instruction Overall (AVG) 2.35 on the 1-5 Gemini-3.1-Pro judge scale, stored as 2.35/5. Open model; default settings not further specified.",
    "source_id": "visual-edit-compass"
  },
  {
    "model": "joyai-image-edit",
    "bench": "edit-compass-en-overall",
    "score": 0.536,
    "source": "https://arxiv.org/html/2605.13062v1#S4.T3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "'JoyAI-Image-Edit Joy Future Academy (2026)'; S4.T3 English-instruction Overall (AVG) 2.68 on the 1-5 Gemini-3.1-Pro judge scale, stored as 2.68/5. Open model; default settings not further specified.",
    "source_id": "visual-edit-compass"
  },
  {
    "model": "anyedit",
    "bench": "weavebench-avg",
    "score": 0.472,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "lab",
    "date": "2025-11-14",
    "note": "'AnyEdit [86] (1B)'; Table 2 WEAVEBench Avg 0.472 (0-1). WEAVE's first author (Wei Chow) is an AnyEdit/AnySD author, so by: lab.",
    "source_id": "visual-weavebench"
  },
  {
    "model": "omnigen-v1",
    "bench": "weavebench-avg",
    "score": 0.404,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'OminiGen [80] (4B)'; Table 2 WEAVEBench Avg 0.404 (0-1). 'OminiGen' is the table's spelling of OmniGen (v1), 4B.",
    "source_id": "visual-weavebench"
  },
  {
    "model": "ovis-u1-3b",
    "bench": "weavebench-avg",
    "score": 0.422,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'Ovis-U1 [67] (3B)'; Table 2 WEAVEBench Avg 0.422 (0-1). ",
    "source_id": "visual-weavebench"
  },
  {
    "model": "skywork-unipic-1-5b",
    "bench": "weavebench-avg",
    "score": 0.511,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'UniPic [68] (1.5B)'; Table 2 WEAVEBench Avg 0.511 (0-1). ",
    "source_id": "visual-weavebench"
  },
  {
    "model": "unipic2-sd3-5m-kontext-2b",
    "bench": "weavebench-avg",
    "score": 0.568,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'UniPic2-SD3.5M [73] (2B)'; Table 2 WEAVEBench Avg 0.568 (0-1). ",
    "source_id": "visual-weavebench"
  },
  {
    "model": "unipic2-metaquery-9b",
    "bench": "weavebench-avg",
    "score": 0.582,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'UniPic2-Metaquery [73] (9B)'; Table 2 WEAVEBench Avg 0.582 (0-1). ",
    "source_id": "visual-weavebench"
  },
  {
    "model": "nextstep-1-large-edit",
    "bench": "weavebench-avg",
    "score": 0.534,
    "source": "https://arxiv.org/html/2511.11434v1#S4.T2",
    "by": "independent",
    "date": "2025-11-14",
    "note": "'NextStep-1-Large [66] (15B)'; Table 2 WEAVEBench Avg 0.534 (0-1). ",
    "source_id": "visual-weavebench"
  },
  {
    "model": "nextstep-1-large-edit",
    "bench": "unicbench-en-overall",
    "score": 0.64076,
    "source": "https://arxiv.org/html/2512.02790v1#S6.T4",
    "by": "independent",
    "date": "2025-12-01",
    "note": "NextStep-1 [34]; S6.T4 Overall-EN 6.4076 on the 0-10 gpt-4.1-2025-04-14 judge scale, stored as 6.4076/10. Open-source model; settings not tabulated.",
    "source_id": "visual-unicbench"
  },
  {
    "model": "ovis-u1-3b",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.243,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'Ovis-U1-3B (Wang et al., 2025a)'; Table 3 Result Metric Avg 24.3% -> /100. Without-CoT block (same column as the imported image-model rows).",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "unipic2-metaquery-9b",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.08,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'UniPic2-M-9B (Wei et al., 2025)'; Table 3 Result Metric Avg 8.0% -> /100. Without-CoT block (same column as the imported image-model rows).",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "hidream-e1-1",
    "bench": "vigor-bench-result-avg-image",
    "score": 0.017,
    "source": "https://arxiv.org/html/2603.25823v1#S3.T3",
    "by": "independent",
    "date": "2026-03-26",
    "note": "'HiDream-E1.1 (Cai et al., 2025)'; Table 3 Result Metric Avg 1.7% -> /100. Without-CoT block (same column as the imported image-model rows).",
    "source_id": "visual-vigor-bench"
  },
  {
    "model": "deepgen-1-0",
    "bench": "chartsync-overall",
    "score": 0.2361,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'DeepGen-1.0 Wang et al. (2026)'; Table 2 ChartSync Overall 23.61 -> 23.61/100; open-source model, inference configuration per Table 7 (S-A3.T7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "flux-2-klein-base-9b",
    "bench": "chartsync-overall",
    "score": 0.5062,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'FLUX.2-Klein-Base-9B Labs (2025)'; Table 2 ChartSync Overall 50.62 -> 50.62/100; open-source model, inference configuration per Table 7 (S-A3.T7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "firered-image-edit-1-0",
    "bench": "chartsync-overall",
    "score": 0.5161,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'FireRed-Image-Edit-1.0 Team et al. (2026)'; Table 2 ChartSync Overall 51.61 -> 51.61/100; open-source model, inference configuration per Table 7 (S-A3.T7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "firered-image-edit-1-1",
    "bench": "chartsync-overall",
    "score": 0.5622,
    "source": "https://arxiv.org/html/2607.10301v1#S4.T2",
    "by": "independent",
    "date": "2026-07-11",
    "note": "'FireRed-Image-Edit-1.1 Team et al. (2026)'; Table 2 ChartSync Overall 56.22 -> 56.22/100; open-source model, inference configuration per Table 7 (S-A3.T7).",
    "source_id": "visual-chartsync"
  },
  {
    "model": "deepgen-1-0",
    "bench": "wise-wiscore-overall",
    "score": 0.73,
    "source": "https://arxiv.org/html/2503.07265v4#A10.T11",
    "by": "independent",
    "date": "2026-06-02",
    "note": "'DeepGen1.0 (Wang et al., 2026)'; WISE v4 Appendix Table 11 (A10.T11) 'Additional evaluation of recent AR and AR+Diffusion models', WiScore 0.73. Sec. 4: 'we additionally evaluate recent autoregressive and AR+Diffusion models in Table 11' (the WISE authors' own supplementary run with the same WiScore metric). Adapted row licensed CC-BY-SA-4.0 (WISE v4 is CC BY-SA 4.0), like the other WISE rows.",
    "source_id": "visual-wise"
  },
  {
    "model": "longcat-image",
    "bench": "wise-wiscore-overall",
    "score": 0.65,
    "source": "https://arxiv.org/html/2503.07265v4#A10.T11",
    "by": "independent",
    "date": "2026-06-02",
    "note": "'LongCat-Image (Team et al., 2025)'; WISE v4 Appendix Table 11 (A10.T11) 'Additional evaluation of recent AR and AR+Diffusion models', WiScore 0.65. Sec. 4: 'we additionally evaluate recent autoregressive and AR+Diffusion models in Table 11' (the WISE authors' own supplementary run with the same WiScore metric). Adapted row licensed CC-BY-SA-4.0 (WISE v4 is CC BY-SA 4.0), like the other WISE rows.",
    "source_id": "visual-wise"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "wise-wiscore-overall",
    "score": 0.57,
    "source": "https://arxiv.org/html/2503.07265v4#A10.T11",
    "by": "independent",
    "date": "2026-06-02",
    "note": "'Hunyuan-Image 3.0 (Cao et al., 2025)'; WISE v4 Appendix Table 11 (A10.T11) 'Additional evaluation of recent AR and AR+Diffusion models', WiScore 0.57. Sec. 4: 'we additionally evaluate recent autoregressive and AR+Diffusion models in Table 11' (the WISE authors' own supplementary run with the same WiScore metric). Adapted row licensed CC-BY-SA-4.0 (WISE v4 is CC BY-SA 4.0), like the other WISE rows.",
    "source_id": "visual-wise"
  },
  {
    "model": "janusflow-1-3b",
    "bench": "mmmg-kg-score-avg",
    "score": 0.0977,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'JanusFlow-1.3B', column 'Avg' = 9.77 (MMMG-Score x100, stored /100); same table and protocol as the imported rows.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "cogview4-6b",
    "bench": "mmmg-kg-score-avg",
    "score": 0.131,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'CogView-4', column 'Avg' = 13.10 (MMMG-Score x100, stored /100); same table and protocol as the imported rows.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "infinity-8b",
    "bench": "mmmg-kg-score-avg",
    "score": 0.1918,
    "source": "https://arxiv.org/html/2506.10963v2",
    "by": "independent",
    "date": "2025-06-13",
    "note": "MMMG (arXiv 2506.10963v2, CC BY-SA 4.0) Table 2, row 'Infinity', column 'Avg' = 19.18 (MMMG-Score x100, stored /100); same table and protocol as the imported rows. Mapped to Infinity-8B: Table 2 lists 1024 px and Sec. 5 names it 'Infinity-8B'.",
    "source_id": "paper-2506.10963v2-mmmg-kg"
  },
  {
    "model": "cogview4-6b",
    "bench": "hpsv3-benchmark",
    "score": 9.61,
    "source": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md",
    "by": "independent",
    "date": "2025-12-05",
    "note": "HPSv3 repository README (MizzenAI/HPSv3 @bd0c5fc, MIT), 'Image Generation Benchmark of HPSv3' table, row 'CogView4', column 'Overall' = 9.61 (raw HPSv3 score; same table as the imported rows).",
    "source_id": "hpsv3-readme-mit"
  },
  {
    "model": "imagen-4-fast",
    "bench": "rapidata-benchai-image-overall",
    "score": 930.2,
    "source": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md#overall-ranking-elo-aggregated-across-all-four-leaderboards",
    "by": "independent",
    "date": "2026-10-01",
    "note": "'Imagen 4 Fast' (Google); Overall ELO 930.2 (rank 32; Alignment 893, Preference 946, Coherence 953, Text coherence 914). 1,514 prompts, human pairwise votes; same table as the imported rows.",
    "source_id": "visual-rapid-benchai-image"
  },
  {
    "model": "imagen-4-fast",
    "bench": "rapidata-face-alignment",
    "score": 1027.69,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'google/imagen-4-fast' (Replicate model id); leaderboard score 1027.690000 (3205 wins / 6180 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "minimax-image-01",
    "bench": "rapidata-face-alignment",
    "score": 1051.62,
    "source": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md#the-resulting-leaderboard",
    "by": "independent",
    "date": "2025-09-03",
    "note": "'minimax/image-01' (Replicate model id); leaderboard score 1051.620000 (2726 wins / 5125 matches). Face-description alignment, human pairwise votes.",
    "source_id": "visual-rapid-face"
  },
  {
    "model": "boogu-image-0-1-base",
    "bench": "ultratext-bench-composite",
    "score": 0.7789,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'Boogu-Image-0.1-Base', column 'Composite' = 77.89 (0-100 VLM rating; stored /100); same table and protocol as the imported rows; open-weight model run by the UltraText authors (Westlake University et al.), who did not build it.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "hidream-o1-image-dev",
    "bench": "ultratext-bench-composite",
    "score": 0.4044,
    "source": "https://arxiv.org/html/2610.09823v1",
    "by": "independent",
    "date": "2026-10-07",
    "note": "UltraText Bench (arXiv 2610.09823v1, 7 Oct 2026) Table 5 leaderboard, row 'HiDream-O1-Image-Dev', column 'Composite' = 40.44 (0-100 VLM rating; stored /100); same table and protocol as the imported rows; open-weight model run by the UltraText authors (Westlake University et al.), who did not build it.",
    "source_id": "paper-2610.09823v1-ultratext"
  },
  {
    "model": "ema-lightning",
    "bench": "freya-tr-eval-wer",
    "score": 0.0092,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "lab",
    "date": "2026-10-06",
    "note": "'EMA Lightning (compiled)'; WER 0.92% -> 0.92/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). The card's own model; compiled and plain PyTorch rows print the same WER.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "trendyol-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0098,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'Trendyol-TTS'; WER 0.98% -> 0.98/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †).",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "gemini-3-8-flash-lite-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0131,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'Gemini 3.8 Flash-Lite'; WER 1.31% -> 1.31/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). Gemini voice Kore.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "gemini-3-8-flash-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0135,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'Gemini 3.8 Flash'; WER 1.35% -> 1.35/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). Gemini voice Kore.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "elevenlabs-v4",
    "bench": "freya-tr-eval-wer",
    "score": 0.0143,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'ElevenLabs v4'; WER 1.43% -> 1.43/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). Default voice George, default settings.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "elevenlabs-v4-turbo",
    "bench": "freya-tr-eval-wer",
    "score": 0.0154,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'ElevenLabs v4 Turbo'; WER 1.54% -> 1.54/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). Default voice George, default settings.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "antalia-1",
    "bench": "freya-tr-eval-wer",
    "score": 0.0688,
    "source": "https://huggingface.co/canberkkkkkk/ema-lightning/blob/7a6ba1ad216bb2f1da9863f80ac8770a6a807632/README.md#benchmarks",
    "by": "independent",
    "date": "2026-10-06",
    "note": "'Antalia 1 ‡'; WER 6.88% -> 6.88/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). EMA Lightning card, 'Our runs' (row without †). Card note: three of its takes failed inside its own code and count as silence.",
    "source_id": "card3-canberkkkkkk-ema-lightning"
  },
  {
    "model": "ema-lightning",
    "bench": "freya-tr-eval-wer",
    "score": 0.0104,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'EMA Lightning (plain PyTorch)'; WER 1.04% -> 1.04/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "antalia-2-mini",
    "bench": "freya-tr-eval-wer",
    "score": 0.0105,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "lab",
    "date": "2026-10-09",
    "note": "'Antalia-2 Mini'; WER 1.05% -> 1.05/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). The card's own model.",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "trendyol-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0116,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'Trendyol-TTS (cfg 2.0, 16 steps)'; WER 1.16% -> 1.16/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "gemini-3-8-flash-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0123,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'Gemini 3.8 Flash TTS (voice Kore)'; WER 1.23% -> 1.23/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). Vertex AI gemini-3.8-flash-tts per harness/synth_api.py.",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "gemini-3-8-flash-lite-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.0134,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'Gemini 3.8 Flash-Lite TTS (voice Kore)'; WER 1.34% -> 1.34/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). Vertex AI gemini-3.8-flash-lite-tts per harness/synth_api.py.",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "piper-tr-tr-dfki-medium",
    "bench": "freya-tr-eval-wer",
    "score": 0.0341,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'Piper (tr_TR dfki medium, CPU)'; WER 3.41% -> 3.41/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "xtts-v2",
    "bench": "freya-tr-eval-wer",
    "score": 0.0367,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'XTTS-v2 (Anka male reference)'; WER 3.67% -> 3.67/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). coqui-tts tts_models/multilingual/multi-dataset/xtts_v2, language tr, cloning the Anka male reference clip (run_xtts.py).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "antalia-1",
    "bench": "freya-tr-eval-wer",
    "score": 0.0628,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "lab",
    "date": "2026-10-09",
    "note": "'Antalia-1 (open weights, release recipe)'; WER 6.28% -> 6.28/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). Patientdesk.ai's own earlier model.",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "mms-tts-tur",
    "bench": "freya-tr-eval-wer",
    "score": 0.0638,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'MMS-TTS (tur)'; WER 6.38% -> 6.38/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "coqui-tr-common-voice-glow-tts",
    "bench": "freya-tr-eval-wer",
    "score": 0.1093,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'Coqui GlowTTS (tr common-voice, lowercased input)'; WER 10.93% -> 10.93/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41). coqui-tts tts_models/tr/common-voice/glow-tts with Turkish lowercasing of the input (run_glow.py).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "freyatts-small",
    "bench": "freya-tr-eval-wer",
    "score": 0.1202,
    "source": "https://huggingface.co/cloud0day3/antalia-mini/blob/1e8166a7436f3e11f7f03b339258ec30f366db60/README.md#on-the-public-benchmark-page",
    "by": "independent",
    "date": "2026-10-09",
    "note": "'FreyaTTS-small (default voice)'; WER 12.02% -> 12.02/100 on Freya-TR-Eval (495 sentences, 3 seeds/takes, 8 kHz band, Whisper large-v3 beam 5, Freya normalisation). Antalia-2 Mini card, 'On the public benchmark page' re-run table (Patientdesk.ai harness, EMA Lightning's eval.py, NVIDIA L4; checkpoints per PatientdeskAI/freya-tr-eval-rerun@d5edf41).",
    "source_id": "card3-cloud0day3-antalia-mini"
  },
  {
    "model": "nano-banana-2",
    "bench": "multitextedit-semavg",
    "score": 0.8012,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'Nano Banana 2'; Table 3 (S5.T3) SemAvg 4.006 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 4.006/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings. Proprietary, official API, highest-quality options.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "gpt-image-1-5",
    "bench": "multitextedit-semavg",
    "score": 0.7154,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'GPT-image-1.5'; Table 3 (S5.T3) SemAvg 3.577 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 3.577/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings. Proprietary, official API, highest-quality options (OpenAI ref: GPT-image-1.5, Dec 16 2025).",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "qwen-image-edit-2511",
    "bench": "multitextedit-semavg",
    "score": 0.6428,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'Qwen-Image-Edit-2511'; Table 3 (S5.T3) SemAvg 3.214 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 3.214/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "firered-image-edit-1-1",
    "bench": "multitextedit-semavg",
    "score": 0.6408,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'FireRed-Image-Edit-1.1'; Table 3 (S5.T3) SemAvg 3.204 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 3.204/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "flux-1-kontext-dev",
    "bench": "multitextedit-semavg",
    "score": 0.4802,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'FLUX.1-Kontext-dev'; Table 3 (S5.T3) SemAvg 2.401 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 2.401/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "bagel",
    "bench": "multitextedit-semavg",
    "score": 0.4358,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'Bagel-thinking'; Table 3 (S5.T3) SemAvg 2.179 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 2.179/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings. Thinking-enabled setting (highest reasoning setting; the non-thinking 'Bagel' row prints 2.424), matching the Bagel-Think choice on Edit-Compass.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "qwen-image-edit",
    "bench": "multitextedit-semavg",
    "score": 0.411,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'Qwen-Image-Edit'; Table 3 (S5.T3) SemAvg 2.055 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 2.055/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "omnigen2",
    "bench": "multitextedit-semavg",
    "score": 0.3468,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'OmniGen2'; Table 3 (S5.T3) SemAvg 1.734 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 1.734/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "instructpix2pix",
    "bench": "multitextedit-semavg",
    "score": 0.238,
    "source": "https://arxiv.org/html/2605.08163v2#S5.T3",
    "by": "independent",
    "date": "2026-05-18",
    "note": "'InstructPix2Pix'; Table 3 (S5.T3) SemAvg 1.190 on the 0-5 LVM-judge scale (mean of IF, TA, VC, LP, LSF over 3,600 instances in 12 languages), stored as 1.190/5. Run by the MultiTextEdit authors with the official pipeline and highest-quality recommended settings.",
    "source_id": "visual-multitextedit-ccby"
  },
  {
    "model": "grok-imagine-video",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.517,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'Grok Imagine ∗'; Table 2 (S4.T2) APEO paired average pAvg 51.7% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Cited as xAI 'Grok imagine' (2025); paper dated May 2026, before Grok Imagine Video 1.5 (June 2026).",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "veo-3-1",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.509,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'Veo 3.1 ∗'; Table 2 (S4.T2) APEO paired average pAvg 50.9% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Cited as the Veo 3.1 technical report; tier not stated.",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "seedance-2-0",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.418,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'Seedance 2.0 ∗'; Table 2 (S4.T2) APEO paired average pAvg 41.8% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Cited as the Seedance 2.0 report (arXiv 2604.14148).",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "seedance-1-5-pro",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.324,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'Seedance 1.5 ∗'; Table 2 (S4.T2) APEO paired average pAvg 32.4% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Cited as 'Seedance 1.5 pro: a native audio-visual joint generation foundation model' (arXiv 2512.13507).",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "wan-2-2-5b",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.284,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'Wan2.2-5B'; Table 2 (S4.T2) APEO paired average pAvg 28.4% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Open weights, public default inference configuration (TI2V-5B is the only 5B Wan2.2 checkpoint).",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "hunyuanvideo-1-5-i2v",
    "bench": "whatif-world-apeo-pavg",
    "score": 0.27,
    "source": "https://arxiv.org/html/2605.27589v1#S4.T2",
    "by": "independent",
    "date": "2026-05-26",
    "note": "'HunyuanVideo1.5-8.3B'; Table 2 (S4.T2) APEO paired average pAvg 27.0% -> /100 (primary paired metric; Gemini 3.1 Pro judge; 319 scenes, two videos per instance from the same initial frame). Open weights, default configuration; every model is conditioned on a real initial frame, so the image-to-video HunyuanVideo-1.5 checkpoint is mapped.",
    "source_id": "visual-whatif-world-ccby"
  },
  {
    "model": "cosmos-predict2-5-2b",
    "bench": "cronos-i2v-success-rate",
    "score": 0.12,
    "source": "https://arxiv.org/html/2605.23699v1#S4.T1",
    "by": "independent",
    "date": "2026-05-22",
    "note": "'Cosmos2.5-2B' (I2V block); Table 1 (S4.T1) Success Rate 0.12 (fraction of videos passing every calibrated per-video metric; averaged over all videos). Cosmos2.5 cited as NVIDIA's 'World simulation with video foundation models for physical AI' (arXiv 2511.00062), i.e. Cosmos-Predict2.5; 'the two available model sizes 2B and 14B'.",
    "source_id": "visual-cronos-ccby"
  },
  {
    "model": "cosmos-predict2-5-14b",
    "bench": "cronos-i2v-success-rate",
    "score": 0.08,
    "source": "https://arxiv.org/html/2605.23699v1#S4.T1",
    "by": "independent",
    "date": "2026-05-22",
    "note": "'Cosmos2.5-14B' (I2V block); Table 1 (S4.T1) Success Rate 0.08 (fraction of videos passing every calibrated per-video metric; averaged over all videos). Cosmos-Predict2.5 14B (see 2B row).",
    "source_id": "visual-cronos-ccby"
  },
  {
    "model": "magi-1-4-5b",
    "bench": "cronos-i2v-success-rate",
    "score": 0.02,
    "source": "https://arxiv.org/html/2605.23699v1#S4.T1",
    "by": "independent",
    "date": "2026-05-22",
    "note": "'MAGI-1-4.5B' (I2V block); Table 1 (S4.T1) Success Rate 0.02 (fraction of videos passing every calibrated per-video metric; averaged over all videos). MAGI-1 [45] 4.5B (base checkpoint as named in Sand AI's model table).",
    "source_id": "visual-cronos-ccby"
  },
  {
    "model": "wan-2-2-i2v-a14b",
    "bench": "cronos-i2v-success-rate",
    "score": 0.2,
    "source": "https://arxiv.org/html/2605.23699v1#S4.T1",
    "by": "independent",
    "date": "2026-05-22",
    "note": "'Wan2.2-14B' (I2V block); Table 1 (S4.T1) Success Rate 0.20 (fraction of videos passing every calibrated per-video metric; averaged over all videos). Wan2.2 14B in the I2V setting, i.e. the Wan2.2-I2V-A14B checkpoint.",
    "source_id": "visual-cronos-ccby"
  },
  {
    "model": "llama-xlam-2-70b-fc-r",
    "bench": "bfcl-v4",
    "score": 0.5307,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'xLAM-2-70b-fc-r (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 53.07%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Salesforce/Llama-xLAM-2-70b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue llama-xlam-2-70b-fc-r (HF Salesforce/Llama-xLAM-2-70b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-xlam-2-70b-fc-r",
    "bench": "bfcl-v3",
    "score": 0.7845,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'xLAM-2-70b-fc-r (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 78.45%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Salesforce/Llama-xLAM-2-70b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue llama-xlam-2-70b-fc-r (HF Salesforce/Llama-xLAM-2-70b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-32b-fc-r",
    "bench": "bfcl-v4",
    "score": 0.5466,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'xLAM-2-32b-fc-r (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 54.66%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Salesforce/xLAM-2-32b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-32b-fc-r (HF Salesforce/xLAM-2-32b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-32b-fc-r",
    "bench": "bfcl-v3",
    "score": 0.7643,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'xLAM-2-32b-fc-r (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 76.43%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Salesforce/xLAM-2-32b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-32b-fc-r (HF Salesforce/xLAM-2-32b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-xlam-2-8b-fc-r",
    "bench": "bfcl-v4",
    "score": 0.4668,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'xLAM-2-8b-fc-r (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 46.68%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Salesforce/Llama-xLAM-2-8b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue llama-xlam-2-8b-fc-r (HF Salesforce/Llama-xLAM-2-8b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-xlam-2-8b-fc-r",
    "bench": "bfcl-v3",
    "score": 0.7204,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'xLAM-2-8b-fc-r (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 72.04%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Salesforce/Llama-xLAM-2-8b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue llama-xlam-2-8b-fc-r (HF Salesforce/Llama-xLAM-2-8b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-3b-fc-r",
    "bench": "bfcl-v4",
    "score": 0.4122,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'xLAM-2-3b-fc-r (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 41.22%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Salesforce/xLAM-2-3b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-3b-fc-r (HF Salesforce/xLAM-2-3b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-3b-fc-r",
    "bench": "bfcl-v3",
    "score": 0.6481,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'xLAM-2-3b-fc-r (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 64.81%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Salesforce/xLAM-2-3b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-3b-fc-r (HF Salesforce/xLAM-2-3b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-1b-fc-r",
    "bench": "bfcl-v4",
    "score": 0.3044,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'xLAM-2-1b-fc-r (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 30.44%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Salesforce/xLAM-2-1b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-1b-fc-r (HF Salesforce/xLAM-2-1b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "xlam-2-1b-fc-r",
    "bench": "bfcl-v3",
    "score": 0.5823,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'xLAM-2-1b-fc-r (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 58.23%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Salesforce/xLAM-2-1b-fc-r (organisation 'Salesforce', licence 'cc-by-nc-4.0'). Identity: catalogue xlam-2-1b-fc-r (HF Salesforce/xLAM-2-1b-fc-r). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "watt-tool-70b",
    "bench": "bfcl-v3",
    "score": 0.7357,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'watt-tool-70B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 73.57%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/watt-ai/watt-tool-70B/ (organisation 'Watt AI Lab', licence 'Apache-2.0'). Identity: catalogue watt-tool-70b (HF watt-ai/watt-tool-70B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "watt-tool-8b",
    "bench": "bfcl-v3",
    "score": 0.6779,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'watt-tool-8B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 67.79%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/watt-ai/watt-tool-8B/ (organisation 'Watt AI Lab', licence 'Apache-2.0'). Identity: catalogue watt-tool-8b (HF watt-ai/watt-tool-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "toolace-2-llama-3-1-8b",
    "bench": "bfcl-v4",
    "score": 0.4244,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'ToolACE-2-8B (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 42.44%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Team-ACE/ToolACE-2-8B (organisation 'Huawei Noah & USTC', licence 'Apache-2.0'). Identity: catalogue toolace-2-llama-3-1-8b (HF Team-ACE/ToolACE-2-Llama-3.1-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "toolace-2-llama-3-1-8b",
    "bench": "bfcl-v3",
    "score": 0.6873,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'ToolACE-2-8B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 68.73%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Team-ACE/ToolACE-2-8B (organisation 'Huawei Noah & USTC', licence 'Apache-2.0'). Identity: catalogue toolace-2-llama-3-1-8b (HF Team-ACE/ToolACE-2-Llama-3.1-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "toolace-8b",
    "bench": "bfcl-v3",
    "score": 0.5835,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'ToolACE-8B (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 58.35%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Team-ACE/ToolACE-8B (organisation 'Huawei Noah & USTC', licence 'Apache-2.0'). Identity: catalogue toolace-8b (HF Team-ACE/ToolACE-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "bitagent-8b",
    "bench": "bfcl-v3",
    "score": 0.6749,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'BitAgent-8B'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 67.49%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/BitAgent/BitAgent-8B/ (organisation 'Bittensor', licence 'Apache-2.0'). Identity: catalogue bitagent-8b (HF BitAgent/BitAgent-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "bitagent-bounty-8b",
    "bench": "bfcl-v4",
    "score": 0.4623,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'BitAgent-Bounty-8B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 46.23%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/BitAgent/BitAgent-Bounty-8B (organisation 'Bittensor', licence 'Apache-2.0'). Identity: catalogue bitagent-bounty-8b (HF BitAgent/BitAgent-Bounty-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "coalm-405b",
    "bench": "bfcl-v3",
    "score": 0.6414,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'CoALM-405B'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 64.14%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/uiuc-convai/CoALM-405B (organisation 'UIUC + Oumi', licence 'Meta Llama 3 Community'). Identity: catalogue coalm-405b (HF uiuc-convai/CoALM-405B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "coalm-70b",
    "bench": "bfcl-v4",
    "score": 0.2799,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'CoALM-70B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 27.99%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/uiuc-convai/CoALM-70B (organisation 'UIUC + Oumi', licence 'Meta Llama 3 Community'). Identity: catalogue coalm-70b (HF uiuc-convai/CoALM-70B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "coalm-70b",
    "bench": "bfcl-v3",
    "score": 0.5449,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'CoALM-70B'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 54.49%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/uiuc-convai/CoALM-70B (organisation 'UIUC + Oumi', licence 'Meta Llama 3 Community'). Identity: catalogue coalm-70b (HF uiuc-convai/CoALM-70B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "coalm-8b",
    "bench": "bfcl-v4",
    "score": 0.2681,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'CoALM-8B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 26.81%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/uiuc-convai/CoALM-8B (organisation 'UIUC + Oumi', licence 'Meta Llama 3 Community'). Identity: catalogue coalm-8b (HF uiuc-convai/CoALM-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "coalm-8b",
    "bench": "bfcl-v3",
    "score": 0.5452,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'CoALM-8B'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 54.52%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/uiuc-convai/CoALM-8B (organisation 'UIUC + Oumi', licence 'Meta Llama 3 Community'). Identity: catalogue coalm-8b (HF uiuc-convai/CoALM-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "functionary-medium-v3-1",
    "bench": "bfcl-v3",
    "score": 0.6232,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Functionary-Medium-v3.1 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 62.32%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/meetkai/functionary-medium-v3.1 (organisation 'MeetKai', licence 'MIT'). Identity: catalogue functionary-medium-v3-1 (HF meetkai/functionary-medium-v3.1). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "functionary-small-v3-1",
    "bench": "bfcl-v3",
    "score": 0.5601,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Functionary-Small-v3.1 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 56.01%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/meetkai/functionary-small-v3.1 (organisation 'MeetKai', licence 'MIT'). Identity: catalogue functionary-small-v3-1 (HF meetkai/functionary-small-v3.1). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-7b",
    "bench": "bfcl-v4",
    "score": 0.3167,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Hammer2.1-7b (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 31.67%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/MadeAgents/Hammer2.1-7b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-7b (HF MadeAgents/Hammer2.1-7b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-7b",
    "bench": "bfcl-v3",
    "score": 0.6225,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Hammer2.1-7b (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 62.25%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/MadeAgents/Hammer2.1-7b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-7b (HF MadeAgents/Hammer2.1-7b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-3b",
    "bench": "bfcl-v4",
    "score": 0.2971,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Hammer2.1-3b (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 29.71%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/MadeAgents/Hammer2.1-3b (organisation 'MadeAgents', licence 'qwen-research'). Identity: catalogue hammer2-1-3b (HF MadeAgents/Hammer2.1-3b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-3b",
    "bench": "bfcl-v3",
    "score": 0.5956,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Hammer2.1-3b (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 59.56%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/MadeAgents/Hammer2.1-3b (organisation 'MadeAgents', licence 'qwen-research'). Identity: catalogue hammer2-1-3b (HF MadeAgents/Hammer2.1-3b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-1-5b",
    "bench": "bfcl-v4",
    "score": 0.2788,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Hammer2.1-1.5b (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 27.88%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/MadeAgents/Hammer2.1-1.5b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-1-5b (HF MadeAgents/Hammer2.1-1.5b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-1-5b",
    "bench": "bfcl-v3",
    "score": 0.5494,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Hammer2.1-1.5b (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 54.94%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/MadeAgents/Hammer2.1-1.5b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-1-5b (HF MadeAgents/Hammer2.1-1.5b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-0-5b",
    "bench": "bfcl-v4",
    "score": 0.2122,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Hammer2.1-0.5b (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 21.22%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/MadeAgents/Hammer2.1-0.5b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-0-5b (HF MadeAgents/Hammer2.1-0.5b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hammer2-1-0-5b",
    "bench": "bfcl-v3",
    "score": 0.4535,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Hammer2.1-0.5b (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 45.35%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/MadeAgents/Hammer2.1-0.5b (organisation 'MadeAgents', licence 'cc-by-nc-4.0'). Identity: catalogue hammer2-1-0-5b (HF MadeAgents/Hammer2.1-0.5b). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "haha-7b",
    "bench": "bfcl-v3",
    "score": 0.5729,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-25/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-25",
    "note": "Published entry 'Haha-7B'. BFCL v3 leaderboard snapshot 2025-04-25 in the BFCL-Result archive; field Overall Acc = 57.29%. Entry absent from the newer v3 snapshot(s) (2025-06-14), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/ZJared/Haha-7B (organisation 'TeleAI', licence 'Apache 2.0'). Identity: catalogue haha-7b (HF ZJared/Haha-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "bfcl-v3",
    "score": 0.5479,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'DeepSeek-Coder-V2 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 54.79%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct-0724 (organisation 'DeepSeek', licence 'DeepSeek License'). Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "cruxeval-zeroeval",
    "score": 0.695,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0724'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8024,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0724'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.32. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "zebralogic-zeroeval",
    "score": 0.205,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0724'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=3.40. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "gsm8k-zeroeval",
    "score": 0.9151,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-coder-0724'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-coder-v2-0724",
    "bench": "aider-code-editing",
    "score": 0.729,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-24",
    "note": "Published entry 'DeepSeek Coder V2 0724'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 72.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.45.2-dev; evaluator commit=89965bf; command=aider --model deepseek/deepseek-coder; run=2024-07-24-07-10-58--deepseek-coder2-0724-diff-direct. Identity: catalogue deepseek-coder-v2-0724 (HF deepseek-ai/DeepSeek-Coder-V2-Instruct-0724). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "firefunction-v1",
    "bench": "bfcl-v3",
    "score": 0.4066,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'FireFunction-v1 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 40.66%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/fireworks-ai/firefunction-v1 (organisation 'Fireworks', licence 'Apache 2.0'). Identity: catalogue firefunction-v1 (HF fireworks-ai/firefunction-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "granite-20b-functioncalling",
    "bench": "bfcl-v4",
    "score": 0.2323,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Granite-20b-FunctionCalling (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 23.23%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/ibm-granite/granite-20b-functioncalling (organisation 'IBM', licence 'Apache-2.0'). Identity: catalogue granite-20b-functioncalling (HF ibm-granite/granite-20b-functioncalling). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "granite-20b-functioncalling",
    "bench": "bfcl-v3",
    "score": 0.4892,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Granite-20b-FunctionCalling (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 48.92%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/ibm-granite/granite-20b-functioncalling (organisation 'IBM', licence 'Apache-2.0'). Identity: catalogue granite-20b-functioncalling (HF ibm-granite/granite-20b-functioncalling). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hermes-2-pro-llama-3-8b",
    "bench": "bfcl-v3",
    "score": 0.4684,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Hermes-2-Pro-Llama-3-8B (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 46.84%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B (organisation 'NousResearch', licence 'apache-2.0'). Identity: catalogue hermes-2-pro-llama-3-8b (HF NousResearch/Hermes-2-Pro-Llama-3-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "hermes-2-pro-llama-3-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6944,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-06",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Hermes-2-Pro-Llama-3-8B' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 69.44 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha cab8e13737. Identity: catalogue hermes-2-pro-llama-3-8b (HF NousResearch/Hermes-2-Pro-Llama-3-8B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "hermes-2-pro-mistral-7b",
    "bench": "bfcl-v3",
    "score": 0.4132,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Hermes-2-Pro-Mistral-7B (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 41.32%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B (organisation 'NousResearch', licence 'apache-2.0'). Identity: catalogue hermes-2-pro-mistral-7b (HF NousResearch/Hermes-2-Pro-Mistral-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "thinkagent-1b",
    "bench": "bfcl-v3",
    "score": 0.2083,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'ThinkAgent-1B (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 20.83%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/ThinkAgents/ThinkAgent-1B (organisation 'ThinkAgents', licence 'apache-2.0'). Identity: catalogue thinkagent-1b (HF AymanTarig/Llama-3.2-1B-FC-v3). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "ling-lite-1-5",
    "bench": "bfcl-v3",
    "score": 0.5452,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'ling-lite-v1.5 (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive; field Overall Acc = 54.52%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/inclusionAI/Ling-lite-1.5 (organisation 'Ling', licence 'MIT'). Identity: catalogue ling-lite-1-5 (HF inclusionAI/Ling-lite-1.5). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nanbeige4-3b-thinking-2511",
    "bench": "bfcl-v4",
    "score": 0.514,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Nanbeige4-3B-Thinking-2511 (FC)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 51.4%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/Nanbeige/Nanbeige4-3B-Thinking-2511 (organisation 'Nanbeige', licence 'apache-2.0'). Identity: catalogue nanbeige4-3b-thinking-2511 (HF Nanbeige/Nanbeige4-3B-Thinking-2511). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nanbeige4-3b-thinking-2511",
    "bench": "eqbench3-elo",
    "score": 1014.33,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'Nanbeige/Nanbeige4-3B-Thinking-2511'; raw TrueSkill rating 'elo' 1014.33 (sigma 6.87, 95% CI 1000.86-1027.8); normalised display value elo_norm not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Identity: catalogue nanbeige4-3b-thinking-2511 (HF Nanbeige/Nanbeige4-3B-Thinking-2511).",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "arch-agent-32b",
    "bench": "bfcl-v4",
    "score": 0.4537,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Arch-Agent-32B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 45.37%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/katanemo/Arch-Agent-32B (organisation 'katanemo', licence 'katanemo-research'). Identity: catalogue arch-agent-32b (HF katanemo/Arch-Agent-32B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "arch-agent-3b",
    "bench": "bfcl-v4",
    "score": 0.3536,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Arch-Agent-3B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 35.36%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/katanemo/Arch-Agent-3B (organisation 'katanemo', licence 'katanemo-research'). Identity: catalogue arch-agent-3b (HF katanemo/Arch-Agent-3B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "arch-agent-1-5b",
    "bench": "bfcl-v4",
    "score": 0.3214,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'Arch-Agent-1.5B'. BFCL V4; 2025-12-16 archive; field Overall Acc = 32.14%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/katanemo/Arch-Agent-1.5B (organisation 'katanemo', licence 'katanemo-research'). Identity: catalogue arch-agent-1-5b (HF katanemo/Arch-Agent-1.5B). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "rzn-t",
    "bench": "bfcl-v4",
    "score": 0.2225,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv",
    "by": "independent",
    "date": "2025-12-16",
    "note": "Published entry 'RZN-T (Prompt)'. BFCL V4; 2025-12-16 archive; field Overall Acc = 22.25%. Native function calling (FC) selected when available, otherwise Prompt; no duplicate component scores. Evaluator release referenced by the official leaderboard: f7cf7359b7ac615a0b294831c5ba2bc95ee4a000 (bfcl-eval 2025.12.17); this is reproduction guidance, not a certified execution commit for each archived row. Aggregate CSV does not record per-run temperature, token limit or weight hash; these are left unknown. Date is archive publication, not necessarily inference date. Model Link https://huggingface.co/phronetic-ai/RZN-T (organisation 'Phronetic AI', licence 'apache-2.0'). Identity: catalogue rzn-t (HF phronetic-ai/RZN-T). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "opencoder-8b-instruct",
    "bench": "humaneval",
    "score": 0.817,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCoder-8B-Instruct' (link https://huggingface.co/infly/OpenCoder-8B-Instruct; chat/instruction prompt); pass@1 humaneval = 81.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencoder-8b-instruct (HF infly/OpenCoder-8B-Instruct). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencoder-8b-instruct",
    "bench": "humaneval-plus",
    "score": 0.774,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCoder-8B-Instruct' (link https://huggingface.co/infly/OpenCoder-8B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 77.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencoder-8b-instruct (HF infly/OpenCoder-8B-Instruct). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencoder-8b-instruct",
    "bench": "mbpp-plus",
    "score": 0.714,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCoder-8B-Instruct' (link https://huggingface.co/infly/OpenCoder-8B-Instruct; chat/instruction prompt); pass@1 mbpp+ = 71.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencoder-8b-instruct (HF infly/OpenCoder-8B-Instruct). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencoder-8b-instruct",
    "bench": "evalperf-correctness-20240328",
    "score": 0.769322033898305,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/infly--OpenCoder-8B-Instruct_vllm_temp_1.0_evalperf_results.brief.json",
    "by": "independent",
    "date": "2024-11-08",
    "note": "Published entry 'infly--OpenCoder-8B-Instruct'. EvalPerf dataset 20240328; correctness pass@1 averaged over 118 tasks; 100 samples/task, temperature=1.0; default prompt (no efficiency instruction); backend=vllm; min_correct=10 and max_profile=20 apply to profiling, not the selected metric. Weight revision hash and token limit not published in this run file. HumanEval+/MBPP+ parent scores, DPS and win rates excluded from this export. Run file date 2024-11-08 20:59. Identity: catalogue opencoder-8b-instruct (HF infly/OpenCoder-8B-Instruct).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "artigenz-coder-ds-6-7b",
    "bench": "humaneval",
    "score": 0.756,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Artigenz-Coder-DS-6.7B' (link https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B; chat/instruction prompt); pass@1 humaneval = 75.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue artigenz-coder-ds-6-7b (HF Artigenz/Artigenz-Coder-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "artigenz-coder-ds-6-7b",
    "bench": "humaneval-plus",
    "score": 0.726,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Artigenz-Coder-DS-6.7B' (link https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B; chat/instruction prompt); pass@1 humaneval+ = 72.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue artigenz-coder-ds-6-7b (HF Artigenz/Artigenz-Coder-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "artigenz-coder-ds-6-7b",
    "bench": "mbpp-plus",
    "score": 0.696,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Artigenz-Coder-DS-6.7B' (link https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B; chat/instruction prompt); pass@1 mbpp+ = 69.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue artigenz-coder-ds-6-7b (HF Artigenz/Artigenz-Coder-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "ajibawa-code-13b",
    "bench": "humaneval",
    "score": 0.561,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-13B' (link https://huggingface.co/ajibawa-2023/Code-13B; chat/instruction prompt); pass@1 humaneval = 56.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue ajibawa-code-13b (HF ajibawa-2023/Code-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "ajibawa-code-13b",
    "bench": "humaneval-plus",
    "score": 0.524,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-13B' (link https://huggingface.co/ajibawa-2023/Code-13B; chat/instruction prompt); pass@1 humaneval+ = 52.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue ajibawa-code-13b (HF ajibawa-2023/Code-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "code-290k-6-7b-instruct",
    "bench": "humaneval",
    "score": 0.646,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-290k-6.7B-Instruct' (link https://huggingface.co/ajibawa-2023/Code-290k-6.7B-Instruct; chat/instruction prompt); pass@1 humaneval = 64.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue code-290k-6-7b-instruct (HF ajibawa-2023/Code-290k-6.7B-Instruct). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "code-290k-6-7b-instruct",
    "bench": "humaneval-plus",
    "score": 0.597,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-290k-6.7B-Instruct' (link https://huggingface.co/ajibawa-2023/Code-290k-6.7B-Instruct; chat/instruction prompt); pass@1 humaneval+ = 59.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue code-290k-6-7b-instruct (HF ajibawa-2023/Code-290k-6.7B-Instruct). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "ajibawa-code-33b",
    "bench": "humaneval",
    "score": 0.549,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-33B' (link https://huggingface.co/ajibawa-2023/Code-33B; chat/instruction prompt); pass@1 humaneval = 54.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue ajibawa-code-33b (HF ajibawa-2023/Code-33B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "ajibawa-code-33b",
    "bench": "humaneval-plus",
    "score": 0.494,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Code-33B' (link https://huggingface.co/ajibawa-2023/Code-33B; chat/instruction prompt); pass@1 humaneval+ = 49.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue ajibawa-code-33b (HF ajibawa-2023/Code-33B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "python-code-13b",
    "bench": "humaneval",
    "score": 0.329,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Python-Code-13B' (link https://huggingface.co/ajibawa-2023/Python-Code-13B; chat/instruction prompt); pass@1 humaneval = 32.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue python-code-13b (HF ajibawa-2023/Python-Code-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "python-code-13b",
    "bench": "humaneval-plus",
    "score": 0.305,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Python-Code-13B' (link https://huggingface.co/ajibawa-2023/Python-Code-13B; chat/instruction prompt); pass@1 humaneval+ = 30.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue python-code-13b (HF ajibawa-2023/Python-Code-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openhermes-2-5-code-290k-13b",
    "bench": "humaneval",
    "score": 0.543,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenHermes-2.5-Code-290k-13B' (link https://huggingface.co/ajibawa-2023/OpenHermes-2.5-Code-290k-13B; chat/instruction prompt); pass@1 humaneval = 54.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue openhermes-2-5-code-290k-13b (HF ajibawa-2023/OpenHermes-2.5-Code-290k-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openhermes-2-5-code-290k-13b",
    "bench": "humaneval-plus",
    "score": 0.488,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenHermes-2.5-Code-290k-13B' (link https://huggingface.co/ajibawa-2023/OpenHermes-2.5-Code-290k-13B; chat/instruction prompt); pass@1 humaneval+ = 48.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue openhermes-2-5-code-290k-13b (HF ajibawa-2023/OpenHermes-2.5-Code-290k-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "openhermes-2-5-code-290k-13b",
    "bench": "mbpp-plus",
    "score": 0.458,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenHermes-2.5-Code-290k-13B' (link https://huggingface.co/ajibawa-2023/OpenHermes-2.5-Code-290k-13B; chat/instruction prompt); pass@1 mbpp+ = 45.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue openhermes-2-5-code-290k-13b (HF ajibawa-2023/OpenHermes-2.5-Code-290k-13B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b",
    "bench": "humaneval",
    "score": 0.518,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B' (link https://huggingface.co/Qwen/CodeQwen1.5-7B; base-model completion prompt (not prompted)); pass@1 humaneval = 51.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue codeqwen1-5-7b (HF Qwen/CodeQwen1.5-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b",
    "bench": "humaneval-plus",
    "score": 0.457,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B' (link https://huggingface.co/Qwen/CodeQwen1.5-7B; base-model completion prompt (not prompted)); pass@1 humaneval+ = 45.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue codeqwen1-5-7b (HF Qwen/CodeQwen1.5-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b",
    "bench": "mbpp-plus",
    "score": 0.608,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'CodeQwen1.5-7B' (link https://huggingface.co/Qwen/CodeQwen1.5-7B; base-model completion prompt (not prompted)); pass@1 mbpp+ = 60.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue codeqwen1-5-7b (HF Qwen/CodeQwen1.5-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "codeqwen1-5-7b",
    "bench": "bigcodebench-complete",
    "score": 0.456,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'CodeQwen1.5 (base)' / checkpoint '7B', column 'Greedy Decoding · Original' = 0.456 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/Qwen/CodeQwen1.5-7B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "humaneval",
    "score": 0.335,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-base' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B; base-model completion prompt (not prompted)); pass@1 humaneval = 33.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue llama-3-8b-base (HF meta-llama/Meta-Llama-3-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "humaneval-plus",
    "score": 0.293,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-base' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B; base-model completion prompt (not prompted)); pass@1 humaneval+ = 29.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue llama-3-8b-base (HF meta-llama/Meta-Llama-3-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "mbpp-plus",
    "score": 0.516,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Llama3-8B-base' (link https://huggingface.co/meta-llama/Meta-Llama-3-8B; base-model completion prompt (not prompted)); pass@1 mbpp+ = 51.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue llama-3-8b-base (HF meta-llama/Meta-Llama-3-8B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.476362,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Meta-Llama-3-8B' (query id meta-llama/Meta-Llama-3-8B, precision BF16, model sha 8cde5ca8380496c9a6cc7ef3a8b46a0372a1d920); Average Accuracy = mean of the 21 published task scores = 47.6362 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-3-8b-base (HF meta-llama/Meta-Llama-3-8B).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "bigcodebench-complete",
    "score": 0.288,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Llama3-base' / checkpoint '8B', column 'Greedy Decoding · Original' = 0.288 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-8B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-70b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.7791,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3-70B' (source=hub, precision=bfloat16, type=pretrained); published average 77.91 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha b4d08b7db4. Identity: catalogue llama-3-70b-base (HF meta-llama/Meta-Llama-3-70B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-70b-base",
    "bench": "bigcodebench-complete",
    "score": 0.433,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Llama3-base' / checkpoint '70B', column 'Greedy Decoding · Original' = 0.433 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/meta-llama/Meta-Llama-3-70B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "llama-3-1-8b-base",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.485381,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Llama-3.1-8B' (query id meta-llama/Llama-3.1-8B, precision BF16, model sha d04e592bb4f6aa9cfee91e2e20afa771667e1d4b); Average Accuracy = mean of the 21 published task scores = 48.5381 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-3-1-8b-base (HF meta-llama/Llama-3.1-8B).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "wavecoder-ultra-6-7b",
    "bench": "humaneval",
    "score": 0.75,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WaveCoder-Ultra-6.7B' (link https://huggingface.co/microsoft/wavecoder-ultra-6.7b; chat/instruction prompt); pass@1 humaneval = 75.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wavecoder-ultra-6-7b (HF microsoft/wavecoder-ultra-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wavecoder-ultra-6-7b",
    "bench": "humaneval-plus",
    "score": 0.695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WaveCoder-Ultra-6.7B' (link https://huggingface.co/microsoft/wavecoder-ultra-6.7b; chat/instruction prompt); pass@1 humaneval+ = 69.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wavecoder-ultra-6-7b (HF microsoft/wavecoder-ultra-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wavecoder-ultra-6-7b",
    "bench": "mbpp-plus",
    "score": 0.635,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WaveCoder-Ultra-6.7B' (link https://huggingface.co/microsoft/wavecoder-ultra-6.7b; chat/instruction prompt); pass@1 mbpp+ = 63.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wavecoder-ultra-6-7b (HF microsoft/wavecoder-ultra-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-cl-7b",
    "bench": "humaneval",
    "score": 0.707,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-CL-7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-CL-7B; chat/instruction prompt); pass@1 humaneval = 70.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue magicoder-s-cl-7b (HF ise-uiuc/Magicoder-S-CL-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-cl-7b",
    "bench": "humaneval-plus",
    "score": 0.677,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-CL-7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-CL-7B; chat/instruction prompt); pass@1 humaneval+ = 67.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue magicoder-s-cl-7b (HF ise-uiuc/Magicoder-S-CL-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "magicoder-s-cl-7b",
    "bench": "mbpp-plus",
    "score": 0.601,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Magicoder-S-CL-7B' (link https://huggingface.co/ise-uiuc/Magicoder-S-CL-7B; chat/instruction prompt); pass@1 mbpp+ = 60.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue magicoder-s-cl-7b (HF ise-uiuc/Magicoder-S-CL-7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-codealpaca-lora",
    "bench": "humaneval",
    "score": 0.482,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-codealpaca-7B' (link https://huggingface.co/Nondzu/Mistral-7B-codealpaca-lora; base-model completion prompt (not prompted)); pass@1 humaneval = 48.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue mistral-7b-codealpaca-lora (HF Nondzu/Mistral-7B-codealpaca-lora). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistral-7b-codealpaca-lora",
    "bench": "humaneval-plus",
    "score": 0.421,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'Mistral-codealpaca-7B' (link https://huggingface.co/Nondzu/Mistral-7B-codealpaca-lora; base-model completion prompt (not prompted)); pass@1 humaneval+ = 42.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue mistral-7b-codealpaca-lora (HF Nondzu/Mistral-7B-codealpaca-lora). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistralhermes-codepro-7b-v1",
    "bench": "humaneval",
    "score": 0.476,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'MistralHermes-CodePro-7B-v1' (link https://huggingface.co/beowolx/MistralHermes-CodePro-7B-v1; chat/instruction prompt); pass@1 humaneval = 47.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue mistralhermes-codepro-7b-v1 (HF beowolx/MistralHermes-CodePro-7B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistralhermes-codepro-7b-v1",
    "bench": "humaneval-plus",
    "score": 0.421,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'MistralHermes-CodePro-7B-v1' (link https://huggingface.co/beowolx/MistralHermes-CodePro-7B-v1; chat/instruction prompt); pass@1 humaneval+ = 42.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue mistralhermes-codepro-7b-v1 (HF beowolx/MistralHermes-CodePro-7B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "mistralhermes-codepro-7b-v1",
    "bench": "mbpp-plus",
    "score": 0.464,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'MistralHermes-CodePro-7B-v1' (link https://huggingface.co/beowolx/MistralHermes-CodePro-7B-v1; chat/instruction prompt); pass@1 mbpp+ = 46.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue mistralhermes-codepro-7b-v1 (HF beowolx/MistralHermes-CodePro-7B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-33b",
    "bench": "humaneval",
    "score": 0.793,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-33B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B; chat/instruction prompt); pass@1 humaneval = 79.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-33b (HF m-a-p/OpenCodeInterpreter-DS-33B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-33b",
    "bench": "humaneval-plus",
    "score": 0.738,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-33B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B; chat/instruction prompt); pass@1 humaneval+ = 73.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-33b (HF m-a-p/OpenCodeInterpreter-DS-33B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-33b",
    "bench": "mbpp-plus",
    "score": 0.685,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-33B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B; chat/instruction prompt); pass@1 mbpp+ = 68.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-33b (HF m-a-p/OpenCodeInterpreter-DS-33B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-6-7b",
    "bench": "humaneval",
    "score": 0.774,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-6.7B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B; chat/instruction prompt); pass@1 humaneval = 77.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-6-7b (HF m-a-p/OpenCodeInterpreter-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-6-7b",
    "bench": "humaneval-plus",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-6.7B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B; chat/instruction prompt); pass@1 humaneval+ = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-6-7b (HF m-a-p/OpenCodeInterpreter-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "opencodeinterpreter-ds-6-7b",
    "bench": "mbpp-plus",
    "score": 0.664,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'OpenCodeInterpreter-DS-6.7B' (link https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B; chat/instruction prompt); pass@1 mbpp+ = 66.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue opencodeinterpreter-ds-6-7b (HF m-a-p/OpenCodeInterpreter-DS-6.7B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starchat2-15b-v0-1",
    "bench": "humaneval",
    "score": 0.738,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starchat2-15b-v0.1' (link https://huggingface.co/HuggingFaceH4/starchat2-15b-v0.1; chat/instruction prompt); pass@1 humaneval = 73.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starchat2-15b-v0-1 (HF HuggingFaceH4/starchat2-15b-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starchat2-15b-v0-1",
    "bench": "humaneval-plus",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starchat2-15b-v0.1' (link https://huggingface.co/HuggingFaceH4/starchat2-15b-v0.1; chat/instruction prompt); pass@1 humaneval+ = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starchat2-15b-v0-1 (HF HuggingFaceH4/starchat2-15b-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starchat2-15b-v0-1",
    "bench": "mbpp-plus",
    "score": 0.646,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starchat2-15b-v0.1' (link https://huggingface.co/HuggingFaceH4/starchat2-15b-v0.1; chat/instruction prompt); pass@1 mbpp+ = 64.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starchat2-15b-v0-1 (HF HuggingFaceH4/starchat2-15b-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b-instruct-v0-1",
    "bench": "humaneval",
    "score": 0.677,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starcoder2-15b-instruct-v0.1' (link https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1; chat/instruction prompt); pass@1 humaneval = 67.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starcoder2-15b-instruct-v0-1 (HF bigcode/starcoder2-15b-instruct-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b-instruct-v0-1",
    "bench": "humaneval-plus",
    "score": 0.604,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starcoder2-15b-instruct-v0.1' (link https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1; chat/instruction prompt); pass@1 humaneval+ = 60.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starcoder2-15b-instruct-v0-1 (HF bigcode/starcoder2-15b-instruct-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b-instruct-v0-1",
    "bench": "mbpp-plus",
    "score": 0.651,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'starcoder2-15b-instruct-v0.1' (link https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1; chat/instruction prompt); pass@1 mbpp+ = 65.1 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starcoder2-15b-instruct-v0-1 (HF bigcode/starcoder2-15b-instruct-v0.1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoder2-15b-instruct-v0-1",
    "bench": "bigcodebench-complete",
    "score": 0.437,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'StarCoder2-Instruct' / checkpoint '15B', column 'Greedy Decoding · Original' = 0.437 (calibrated 0.451). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/bigcode/starcoder2-15b-instruct-v0.1. Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.' StarCoder2-Instruct is a BigCode model and several BigCodeBench authors are BigCode/StarCoder2 authors, so by: lab (as for the catalogued StarCoder2 base rows).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "starcoder2-15b-instruct-v0-1",
    "bench": "bigcodebench-instruct",
    "score": 0.367,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'StarCoder2-Instruct' / checkpoint '15B', column 'Greedy Decoding · Original' = 0.367 (calibrated 0.376). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/bigcode/starcoder2-15b-instruct-v0.1. Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.' StarCoder2-Instruct is a BigCode model and several BigCodeBench authors are BigCode/StarCoder2 authors, so by: lab (as for the catalogued StarCoder2 base rows).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "starcoderbase-1b",
    "bench": "humaneval",
    "score": 0.146,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-1B' (link https://huggingface.co/bigcode/starcoderbase-1b; base-model completion prompt (not prompted)); pass@1 humaneval = 14.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starcoderbase-1b (HF bigcode/starcoderbase-1b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "starcoderbase-1b",
    "bench": "humaneval-plus",
    "score": 0.122,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'StarCoderBase-1B' (link https://huggingface.co/bigcode/starcoderbase-1b; base-model completion prompt (not prompted)); pass@1 humaneval+ = 12.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue starcoderbase-1b (HF bigcode/starcoderbase-1b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "whiterabbitneo-33b-v1",
    "bench": "humaneval",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WhiteRabbitNeo-33B-v1' (link https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1; chat/instruction prompt); pass@1 humaneval = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue whiterabbitneo-33b-v1 (HF WhiteRabbitNeo/WhiteRabbitNeo-33B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "whiterabbitneo-33b-v1",
    "bench": "humaneval-plus",
    "score": 0.659,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WhiteRabbitNeo-33B-v1' (link https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1; chat/instruction prompt); pass@1 humaneval+ = 65.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue whiterabbitneo-33b-v1 (HF WhiteRabbitNeo/WhiteRabbitNeo-33B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "whiterabbitneo-33b-v1",
    "bench": "mbpp-plus",
    "score": 0.669,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WhiteRabbitNeo-33B-v1' (link https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1; chat/instruction prompt); pass@1 mbpp+ = 66.9 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue whiterabbitneo-33b-v1 (HF WhiteRabbitNeo/WhiteRabbitNeo-33B-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-15b-v1-0",
    "bench": "humaneval",
    "score": 0.567,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-15B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-15B-V1.0; chat/instruction prompt); pass@1 humaneval = 56.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wizardcoder-15b-v1-0 (HF WizardLMTeam/WizardCoder-15B-V1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-15b-v1-0",
    "bench": "humaneval-plus",
    "score": 0.506,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-15B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-15B-V1.0; chat/instruction prompt); pass@1 humaneval+ = 50.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wizardcoder-15b-v1-0 (HF WizardLMTeam/WizardCoder-15B-V1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "wizardcoder-15b-v1-0",
    "bench": "mbpp-plus",
    "score": 0.542,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'WizardCoder-15B-V1.0' (link https://huggingface.co/WizardLM/WizardCoder-15B-V1.0; chat/instruction prompt); pass@1 mbpp+ = 54.2 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue wizardcoder-15b-v1-0 (HF WizardLMTeam/WizardCoder-15B-V1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xwincoder-34b",
    "bench": "humaneval",
    "score": 0.756,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'XwinCoder-34B' (link https://huggingface.co/Xwin-LM/XwinCoder-34B; chat/instruction prompt); pass@1 humaneval = 75.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xwincoder-34b (HF Xwin-LM/XwinCoder-34B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xwincoder-34b",
    "bench": "humaneval-plus",
    "score": 0.695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'XwinCoder-34B' (link https://huggingface.co/Xwin-LM/XwinCoder-34B; chat/instruction prompt); pass@1 humaneval+ = 69.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xwincoder-34b (HF Xwin-LM/XwinCoder-34B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xwincoder-34b",
    "bench": "mbpp-plus",
    "score": 0.648,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'XwinCoder-34B' (link https://huggingface.co/Xwin-LM/XwinCoder-34B; chat/instruction prompt); pass@1 mbpp+ = 64.8 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xwincoder-34b (HF Xwin-LM/XwinCoder-34B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "code-millenials-34b",
    "bench": "humaneval",
    "score": 0.744,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'code-millenials-34B' (link https://huggingface.co/budecosystem/code-millenials-34b; chat/instruction prompt); pass@1 humaneval = 74.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue code-millenials-34b (HF budecosystem/code-millenials-34b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "code-millenials-34b",
    "bench": "humaneval-plus",
    "score": 0.707,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'code-millenials-34B' (link https://huggingface.co/budecosystem/code-millenials-34b; chat/instruction prompt); pass@1 humaneval+ = 70.7 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue code-millenials-34b (HF budecosystem/code-millenials-34b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "code-millenials-34b",
    "bench": "mbpp-plus",
    "score": 0.646,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'code-millenials-34B' (link https://huggingface.co/budecosystem/code-millenials-34b; chat/instruction prompt); pass@1 mbpp+ = 64.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue code-millenials-34b (HF budecosystem/code-millenials-34b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dolphin-2-6-mixtral-8x7b",
    "bench": "humaneval",
    "score": 0.64,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'dolphin-2.6-mixtral-8x7b' (link https://huggingface.co/cognitivecomputations/dolphin-2.6-mixtral-8x7b; chat/instruction prompt); pass@1 humaneval = 64.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue dolphin-2-6-mixtral-8x7b (HF dphn/dolphin-2.6-mixtral-8x7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dolphin-2-6-mixtral-8x7b",
    "bench": "humaneval-plus",
    "score": 0.573,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'dolphin-2.6-mixtral-8x7b' (link https://huggingface.co/cognitivecomputations/dolphin-2.6-mixtral-8x7b; chat/instruction prompt); pass@1 humaneval+ = 57.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue dolphin-2-6-mixtral-8x7b (HF dphn/dolphin-2.6-mixtral-8x7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "dolphin-2-6-mixtral-8x7b",
    "bench": "mbpp-plus",
    "score": 0.59,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'dolphin-2.6-mixtral-8x7b' (link https://huggingface.co/cognitivecomputations/dolphin-2.6-mixtral-8x7b; chat/instruction prompt); pass@1 mbpp+ = 59.0 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue dolphin-2-6-mixtral-8x7b (HF dphn/dolphin-2.6-mixtral-8x7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-code-mistral-7b-v1-0",
    "bench": "humaneval",
    "score": 0.482,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-code-mistral-7B-v1.0' (link https://huggingface.co/uukuguy/speechless-code-mistral-7b-v1.0; chat/instruction prompt); pass@1 humaneval = 48.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-code-mistral-7b-v1-0 (HF uukuguy/speechless-code-mistral-7b-v1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-code-mistral-7b-v1-0",
    "bench": "humaneval-plus",
    "score": 0.415,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-code-mistral-7B-v1.0' (link https://huggingface.co/uukuguy/speechless-code-mistral-7b-v1.0; chat/instruction prompt); pass@1 humaneval+ = 41.5 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-code-mistral-7b-v1-0 (HF uukuguy/speechless-code-mistral-7b-v1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-code-mistral-7b-v1-0",
    "bench": "mbpp-plus",
    "score": 0.487,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-code-mistral-7B-v1.0' (link https://huggingface.co/uukuguy/speechless-code-mistral-7b-v1.0; chat/instruction prompt); pass@1 mbpp+ = 48.7 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-code-mistral-7b-v1-0 (HF uukuguy/speechless-code-mistral-7b-v1.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-code-mistral-7b-v1-0",
    "bench": "open-pt-llm-avg9",
    "score": 0.6442,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-06-16",
    "note": "Open PT-LLM Leaderboard row 'uukuguy/speechless-code-mistral-7b-v1.0' (source=hub, precision=float16, type=fine-tuned/fp on domain-specific datasets); published average 64.42 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 1862e0a712. Identity: catalogue speechless-code-mistral-7b-v1-0 (HF uukuguy/speechless-code-mistral-7b-v1.0).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "speechless-codellama-34b-v2-0",
    "bench": "humaneval",
    "score": 0.774,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-codellama-34B-v2.0' (link https://huggingface.co/uukuguy/speechless-codellama-34b-v2.0; chat/instruction prompt); pass@1 humaneval = 77.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-codellama-34b-v2-0 (HF uukuguy/speechless-codellama-34b-v2.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-codellama-34b-v2-0",
    "bench": "humaneval-plus",
    "score": 0.72,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-codellama-34B-v2.0' (link https://huggingface.co/uukuguy/speechless-codellama-34b-v2.0; chat/instruction prompt); pass@1 humaneval+ = 72.0 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-codellama-34b-v2-0 (HF uukuguy/speechless-codellama-34b-v2.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-codellama-34b-v2-0",
    "bench": "mbpp-plus",
    "score": 0.614,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-codellama-34B-v2.0' (link https://huggingface.co/uukuguy/speechless-codellama-34b-v2.0; chat/instruction prompt); pass@1 mbpp+ = 61.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-codellama-34b-v2-0 (HF uukuguy/speechless-codellama-34b-v2.0). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coder-ds-6-7b",
    "bench": "humaneval",
    "score": 0.713,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coder-ds-6.7B' (link https://huggingface.co/uukuguy/speechless-coder-ds-6.7b; chat/instruction prompt); pass@1 humaneval = 71.3 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coder-ds-6-7b (HF uukuguy/speechless-coder-ds-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coder-ds-6-7b",
    "bench": "humaneval-plus",
    "score": 0.659,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coder-ds-6.7B' (link https://huggingface.co/uukuguy/speechless-coder-ds-6.7b; chat/instruction prompt); pass@1 humaneval+ = 65.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coder-ds-6-7b (HF uukuguy/speechless-coder-ds-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coder-ds-6-7b",
    "bench": "mbpp-plus",
    "score": 0.644,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coder-ds-6.7B' (link https://huggingface.co/uukuguy/speechless-coder-ds-6.7b; chat/instruction prompt); pass@1 mbpp+ = 64.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coder-ds-6-7b (HF uukuguy/speechless-coder-ds-6.7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coding-7b-16k-tora",
    "bench": "humaneval",
    "score": 0.549,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coding-7B-16k-tora' (link https://huggingface.co/uukuguy/speechless-coding-7b-16k-tora; chat/instruction prompt); pass@1 humaneval = 54.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coding-7b-16k-tora (HF uukuguy/speechless-coding-7b-16k-tora). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coding-7b-16k-tora",
    "bench": "humaneval-plus",
    "score": 0.506,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coding-7B-16k-tora' (link https://huggingface.co/uukuguy/speechless-coding-7b-16k-tora; chat/instruction prompt); pass@1 humaneval+ = 50.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coding-7b-16k-tora (HF uukuguy/speechless-coding-7b-16k-tora). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-coding-7b-16k-tora",
    "bench": "mbpp-plus",
    "score": 0.506,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-coding-7B-16k-tora' (link https://huggingface.co/uukuguy/speechless-coding-7b-16k-tora; chat/instruction prompt); pass@1 mbpp+ = 50.6 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-coding-7b-16k-tora (HF uukuguy/speechless-coding-7b-16k-tora). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-15b",
    "bench": "humaneval",
    "score": 0.671,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-15b' (link https://huggingface.co/uukuguy/speechless-starcoder2-15b; chat/instruction prompt); pass@1 humaneval = 67.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-15b (HF uukuguy/speechless-starcoder2-15b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-15b",
    "bench": "humaneval-plus",
    "score": 0.628,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-15b' (link https://huggingface.co/uukuguy/speechless-starcoder2-15b; chat/instruction prompt); pass@1 humaneval+ = 62.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-15b (HF uukuguy/speechless-starcoder2-15b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-15b",
    "bench": "mbpp-plus",
    "score": 0.624,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-15b' (link https://huggingface.co/uukuguy/speechless-starcoder2-15b; chat/instruction prompt); pass@1 mbpp+ = 62.4 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-15b (HF uukuguy/speechless-starcoder2-15b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-7b",
    "bench": "humaneval",
    "score": 0.561,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-7b' (link https://huggingface.co/uukuguy/speechless-starcoder2-7b; chat/instruction prompt); pass@1 humaneval = 56.1 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-7b (HF uukuguy/speechless-starcoder2-7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-7b",
    "bench": "humaneval-plus",
    "score": 0.518,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-7b' (link https://huggingface.co/uukuguy/speechless-starcoder2-7b; chat/instruction prompt); pass@1 humaneval+ = 51.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-7b (HF uukuguy/speechless-starcoder2-7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "speechless-starcoder2-7b",
    "bench": "mbpp-plus",
    "score": 0.563,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'speechless-starcoder2-7b' (link https://huggingface.co/uukuguy/speechless-starcoder2-7b; chat/instruction prompt); pass@1 mbpp+ = 56.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue speechless-starcoder2-7b (HF uukuguy/speechless-starcoder2-7b). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xdan-l1-chat-rl-v1",
    "bench": "humaneval",
    "score": 0.402,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'xDAN-L1-Chat-RL-v1-7B' (link https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1; chat/instruction prompt); pass@1 humaneval = 40.2 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xdan-l1-chat-rl-v1 (HF xDAN-AI/xDAN-L1-Chat-RL-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xdan-l1-chat-rl-v1",
    "bench": "humaneval-plus",
    "score": 0.329,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'xDAN-L1-Chat-RL-v1-7B' (link https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1; chat/instruction prompt); pass@1 humaneval+ = 32.9 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xdan-l1-chat-rl-v1 (HF xDAN-AI/xDAN-L1-Chat-RL-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "xdan-l1-chat-rl-v1",
    "bench": "mbpp-plus",
    "score": 0.413,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'xDAN-L1-Chat-RL-v1-7B' (link https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1; chat/instruction prompt); pass@1 mbpp+ = 41.3 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue xdan-l1-chat-rl-v1 (HF xDAN-AI/xDAN-L1-Chat-RL-v1). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zyte-1b",
    "bench": "humaneval",
    "score": 0.024,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'zyte-1B' (link https://huggingface.co/aihub-app/zyte-1B; chat/instruction prompt); pass@1 humaneval = 2.4 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval base tests (original 164 problems and tests). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue zyte-1b (HF venkycs/Zyte-1B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zyte-1b",
    "bench": "humaneval-plus",
    "score": 0.018,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'zyte-1B' (link https://huggingface.co/aihub-app/zyte-1B; chat/instruction prompt); pass@1 humaneval+ = 1.8 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue zyte-1b (HF venkycs/Zyte-1B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "zyte-1b",
    "bench": "mbpp-plus",
    "score": 0.075,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'zyte-1B' (link https://huggingface.co/aihub-app/zyte-1B; chat/instruction prompt); pass@1 mbpp+ = 7.5 (greedy decoding, per the leaderboard notes), divided by 100. MBPP+ (EvalPlus expanded tests, v0.2.0). MBPP base column (399-task subset of MBPP-sanitized, not the original MBPP test set) not used. Per-entry evaluation date not published; date is the data file's last change. Identity: catalogue zyte-1b (HF venkycs/Zyte-1B). Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "llama-3-instruct-8b-simpo-v0-2",
    "bench": "open-pt-llm-avg9",
    "score": 0.6841,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-10",
    "note": "Open PT-LLM Leaderboard row 'princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 68.41 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 2b18bdce3e. Identity: catalogue llama-3-instruct-8b-simpo-v0-2 (HF princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-instruct-8b-simpo-v0-2",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.5522,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3-Instruct-8B-SimPO-v0.2'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.19. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue llama-3-instruct-8b-simpo-v0-2 (HF princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-instruct-8b-simpo-v0-2",
    "bench": "gsm8k-zeroeval",
    "score": 0.5754,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'Llama-3-Instruct-8B-SimPO-v0.2'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=2.05. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue llama-3-instruct-8b-simpo-v0-2 (HF princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "llama-3-instruct-8b-simpo-v0-2",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 37.1554,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-3-Instruct-8B-SimPO-v0.2': WB_score.task_macro = 37.1554 (WB_score 41.5054). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue llama-3-instruct-8b-simpo-v0-2 (HF princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-3-instruct-8b-simpo",
    "bench": "alpacaeval2-lc",
    "score": 0.4465,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama-3-Instruct-8B-SimPO' (mode 'community'), length_controlled_winrate = 44.65 (LC standard error 0.88); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 40.53; average length 1825. Config models_configs/Llama-3-Instruct-8B-SimPO/configs.yaml: model_name 'princeton-nlp/Llama-3-Instruct-8B-SimPO', fn_completions 'vllm_local_completions'. Identity: catalogue llama-3-instruct-8b-simpo (HF princeton-nlp/Llama-3-Instruct-8B-SimPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-3-instruct-8b-simpo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6831,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-29",
    "note": "Open PT-LLM Leaderboard row 'princeton-nlp/Llama-3-Instruct-8B-SimPO' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 68.31 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 7e47ec6613. Identity: catalogue llama-3-instruct-8b-simpo (HF princeton-nlp/Llama-3-Instruct-8B-SimPO).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-instruct-8b-simpo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 37.0497,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-3-Instruct-8B-SimPO': WB_score.task_macro = 37.0497 (WB_score 41.1730). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue llama-3-instruct-8b-simpo (HF princeton-nlp/Llama-3-Instruct-8B-SimPO).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gemma-2-9b-it-dpo",
    "bench": "alpacaeval2-lc",
    "score": 0.6766,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gemma-2-9b-it-DPO' (mode 'community'), length_controlled_winrate = 67.66 (LC standard error 0.66); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 65.36; average length 2016. Config models_configs/gemma-2-9b-it-DPO/configs.yaml: model_name 'princeton-nlp/gemma-2-9b-it-DPO', fn_completions 'vllm_local_completions'. Identity: catalogue gemma-2-9b-it-dpo (HF princeton-nlp/gemma-2-9b-it-DPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gemma-2-9b-it-dpo",
    "bench": "open-pt-llm-avg9",
    "score": 0.7042,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-09-19",
    "note": "Open PT-LLM Leaderboard row 'princeton-nlp/gemma-2-9b-it-DPO' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.42 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f646c99fc3. Identity: catalogue gemma-2-9b-it-dpo (HF princeton-nlp/gemma-2-9b-it-DPO).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemma-2-9b-it-dpo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 53.223,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'gemma-2-9b-it-DPO': WB_score.task_macro = 53.2230 (WB_score 54.2578). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue gemma-2-9b-it-dpo (HF princeton-nlp/gemma-2-9b-it-DPO).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "cruxeval-zeroeval",
    "score": 0.705,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-chat-0628'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8081,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-chat-0628'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.11. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "zebralogic-zeroeval",
    "score": 0.227,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-chat-0628'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=5.20. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "gsm8k-zeroeval",
    "score": 0.9393,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'deepseek-v2-chat-0628'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 53.9943,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'deepseek-v2-chat-0628': WB_score.task_macro = 53.9943 (WB_score 53.8086). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "deepseek-v2-chat-0628",
    "bench": "aider-code-editing",
    "score": 0.699,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-07-19",
    "note": "Published entry 'DeepSeek Chat V2 0628'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 69.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.45.2-dev; evaluator commit=96ff06e-dirty; command=aider --model deepseek/deepseek-chat; run=2024-07-19-08-57-13--openrouter-deepseek-chat-v2-0628. Identity: catalogue deepseek-v2-chat-0628 (HF deepseek-ai/DeepSeek-V2-Chat-0628). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "vicuna-7b-v1-5-16k",
    "bench": "livebench-20240726",
    "score": 0.145,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'vicuna-7b-v1.5-16k'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 14.50. Category means: math 9.0, coding 2.6, reasoning 15.3, language 7.9, instruction_following 42.1, data_analysis 9.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue vicuna-7b-v1-5-16k (HF lmsys/vicuna-7b-v1.5-16k).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "yi-6b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6005,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-6B-Chat' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 60.05 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f14752bd87. Identity: catalogue yi-6b-chat (HF 01-ai/Yi-6B-Chat).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-6b-chat",
    "bench": "livebench-20240726",
    "score": 0.0958,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'yi-6b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 9.58. Category means: math 8.5, coding 2.0, reasoning 10.7, language 4.7, instruction_following 27.2, data_analysis 4.4. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue yi-6b-chat (HF 01-ai/Yi-6B-Chat).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "openhands-lm-32b-v0-1",
    "bench": "aider-polyglot",
    "score": 0.102,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-04-19",
    "note": "Published entry 'openhands-lm-32b-v0.1'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.82.2.dev; evaluator commit=c08336f; command=aider --model openrouter/all-hands/openhands-lm-32b-v0.1; run=2025-04-19-14-43-04--o4-mini-patch. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Identity: catalogue openhands-lm-32b-v0-1 (HF OpenHands/openhands-lm-32b-v0.1). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "dracarys2-72b-instruct",
    "bench": "aider-code-editing",
    "score": 0.669,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-05",
    "note": "Published entry 'Dracarys2-72B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 66.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=04a2cbb; command=(via glhf.chat); run=2024-10-05-20-03-10--dracarys-glhf-whole. Identity: catalogue dracarys2-72b-instruct (HF abacusai/Dracarys2-72B-Instruct). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "internlm2-chat-20b",
    "bench": "alpacaeval2-lc",
    "score": 0.1875,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'internlm2-chat-20b-ppo' (mode 'community'), length_controlled_winrate = 18.75 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 21.75; average length 2373. Config models_configs/internlm2-chat-20b-ppo/configs.yaml: model_name 'internlm/internlm2-chat-20b', fn_completions 'huggingface_local_completions'. Identity: catalogue internlm2-chat-20b (HF internlm/internlm2-chat-20b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "internlm2-chat-20b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7059,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Open PT-LLM Leaderboard row 'internlm/internlm2-chat-20b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.59 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 477d474832. Identity: catalogue internlm2-chat-20b (HF internlm/internlm2-chat-20b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "internlm2-chat-20b",
    "bench": "cptub-speakleash-avg",
    "score": 0.429744,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'internlm/internlm2-chat-20b'; Space Average = mean(Sentiment 3.301, Language understanding 2.785, Phraseology 2.385, Tricky questions 0.124) = 2.1487 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue internlm2-chat-20b (HF internlm/internlm2-chat-20b).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "internlm2-chat-20b",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.244,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'internlm2-20b-chat' score 24.4 (95% CI -2.0, 2.2; average #tokens 667); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers. Identity: catalogue internlm2-chat-20b (HF internlm/internlm2-chat-20b).",
    "source_id": "arena-hard-auto-readme-v0-1"
  },
  {
    "model": "snorkel-mistral-pairrm-dpo",
    "bench": "alpacaeval2-lc",
    "score": 0.2639,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Snorkel-Mistral-PairRM-DPO' (mode 'community'), length_controlled_winrate = 26.39 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 30.22; average length 2736. Config models_configs/Snorkel-Mistral-PairRM-DPO/configs.yaml: model_name 'snorkelai/Snorkel-Mistral-PairRM-DPO', fn_completions 'vllm_local_completions'. Identity: catalogue snorkel-mistral-pairrm-dpo (HF snorkelai/Snorkel-Mistral-PairRM-DPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "snorkel-mistral-pairrm-dpo",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.207,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'Snorkel-Mistral-PairRM-DPO' score 20.7 (95% CI -1.8, 2.2; average #tokens 564); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Run and published by the LMSYS/LMArena maintainers. Identity: catalogue snorkel-mistral-pairrm-dpo (HF snorkelai/Snorkel-Mistral-PairRM-DPO).",
    "source_id": "arena-hard-auto-readme-v0-1"
  },
  {
    "model": "hermes-2-theta-llama-3-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Hermes-2-Theta-Llama-3-8B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.0 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha dd2bfa0133. Identity: catalogue hermes-2-theta-llama-3-8b (HF NousResearch/Hermes-2-Theta-Llama-3-8B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "hermes-2-theta-llama-3-8b",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 29.6352,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Hermes-2-Theta-Llama-3-8B': WB_score.task_macro = 29.6352 (WB_score 32.9423). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue hermes-2-theta-llama-3-8b (HF NousResearch/Hermes-2-Theta-Llama-3-8B).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-3-8b-magpie-align-v0-1",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 39.2902,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-3-8B-Magpie-Align-v0.1': WB_score.task_macro = 39.2902 (WB_score 42.4462). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue llama-3-8b-magpie-align-v0-1 (HF Magpie-Align/Llama-3-8B-Magpie-Align-v0.1).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "llama-3-instruct-8b-simpo-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.4578,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama-3-Instruct-8B-SimPO-ExPO' (mode 'community'), length_controlled_winrate = 45.78 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 40.63; average length 1765. Config models_configs/Llama-3-Instruct-8B-SimPO-ExPO/configs.yaml: model_name 'chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue llama-3-instruct-8b-simpo-expo (HF chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-3-instruct-8b-simpo-expo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6771,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-29",
    "note": "Open PT-LLM Leaderboard row 'chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 67.71 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 6438f7ba90. Identity: catalogue llama-3-instruct-8b-simpo-expo (HF chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-instruct-8b-simpo-expo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 35.015,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Llama-3-Instruct-8B-SimPO-ExPO': WB_score.task_macro = 35.0150 (WB_score 39.6872). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue llama-3-instruct-8b-simpo-expo (HF chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "starling-lm-7b-beta-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.2641,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Starling-LM-7B-beta-ExPO' (mode 'community'), length_controlled_winrate = 26.41 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 29.60; average length 2215. Config models_configs/Starling-LM-7B-beta-ExPO/configs.yaml: model_name 'chujiezheng/Starling-LM-7B-beta-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue starling-lm-7b-beta-expo (HF chujiezheng/Starling-LM-7B-beta-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "starling-lm-7b-beta-expo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 31.5594,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'Starling-LM-7B-beta-ExPO': WB_score.task_macro = 31.5594 (WB_score 35.0147). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue starling-lm-7b-beta-expo (HF chujiezheng/Starling-LM-7B-beta-ExPO).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "neo-7b-instruct-v0-1",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 25.0192,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'neo_7b_instruct_v0.1': WB_score.task_macro = 25.0192 (WB_score 29.1992). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue neo-7b-instruct-v0-1 (HF m-a-p/neo_7b_instruct_v0.1).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "neo-7b-instruct-v0-1-expo",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 23.1142,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'neo_7b_instruct_v0.1-ExPO': WB_score.task_macro = 23.1142 (WB_score 27.6246). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue neo-7b-instruct-v0-1-expo (HF chujiezheng/neo_7b_instruct_v0.1-ExPO).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "bigcodebench-complete",
    "score": 0.438,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code-Instruct' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.438 (calibrated 0.444). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-34b-code-instruct (HF API now redirects to ibm-granite/granite-34b-code-instruct-8k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-34b-code-instruct-8k",
    "bench": "bigcodebench-instruct",
    "score": 0.358,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Granite-Code-Instruct' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.358 (calibrated 0.361). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-34b-code-instruct (HF API now redirects to ibm-granite/granite-34b-code-instruct-8k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-3b-code-instruct-2k",
    "bench": "bigcodebench-complete",
    "score": 0.315,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code-Instruct' / checkpoint '3B', column 'Greedy Decoding · Original' = 0.315 (calibrated 0.315). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-3b-code-instruct (HF API now redirects to ibm-granite/granite-3b-code-instruct-2k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-34b-code-base-8k",
    "bench": "bigcodebench-complete",
    "score": 0.385,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code (base)' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.385 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-34b-code-base (HF API now redirects to ibm-granite/granite-34b-code-base-8k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "granite-3b-code-base-2k",
    "bench": "bigcodebench-complete",
    "score": 0.202,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Granite-Code (base)' / checkpoint '3B', column 'Greedy Decoding · Original' = 0.202 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/ibm-granite/granite-3b-code-base (HF API now redirects to ibm-granite/granite-3b-code-base-2k, the catalogue repo). Table 6 caption: 'Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.'",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-34b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.7204,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-15",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-34B' (source=hub, precision=bfloat16, type=pretrained); published average 72.04 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 81136a42ef. Identity: catalogue yi-1-5-34b-base (HF 01-ai/Yi-1.5-34B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-1-5-34b-base",
    "bench": "bigcodebench-complete",
    "score": 0.4,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5 (base)' / checkpoint '34B', column 'Greedy Decoding · Original' = 0.4 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-34B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.6807,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-15",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-9B' (source=hub, precision=bfloat16, type=pretrained); published average 68.07 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 9a6839c5b9. Identity: catalogue yi-1-5-9b-base (HF 01-ai/Yi-1.5-9B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "bigcodebench-complete",
    "score": 0.355,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5 (base)' / checkpoint '9B', column 'Greedy Decoding · Original' = 0.355 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-9B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "yi-1-5-6b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.6236,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-6B' (source=hub, precision=bfloat16, type=pretrained); published average 62.36 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a548af01e9. Identity: catalogue yi-1-5-6b-base (HF 01-ai/Yi-1.5-6B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-1-5-6b-base",
    "bench": "bigcodebench-complete",
    "score": 0.274,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Yi-1.5 (base)' / checkpoint '6B', column 'Greedy Decoding · Original' = 0.274 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/01-ai/Yi-1.5-6B.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mixtral-8x22b-v0-1",
    "bench": "bigcodebench-complete",
    "score": 0.455,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Mixtral-base' / checkpoint '8x22B', column 'Greedy Decoding · Original' = 0.455 (no calibrated value for base models). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4: huggingface.co/mistralai/Mixtral-8x22B-v0.1.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "glm-4-9b-chat-1m",
    "bench": "open-pt-llm-avg9",
    "score": 0.741,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-06-05",
    "note": "Open PT-LLM Leaderboard row 'THUDM/glm-4-9b-chat-1m' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 74.1 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha d74db5ffe6. Identity: catalogue glm-4-9b-chat-1m (HF zai-org/glm-4-9b-chat-1m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "glm-4-9b-chat-1m",
    "bench": "ruler-64k",
    "score": 0.867,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GLM4' (link https://huggingface.co/THUDM/glm-4-9b-chat-1m; claimed length 1M, effective length 64K), column '64K' = 86.7: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 64K column. Identity: catalogue glm-4-9b-chat-1m (HF zai-org/glm-4-9b-chat-1m).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "glm-4-9b-chat-1m",
    "bench": "ruler-128k",
    "score": 0.831,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GLM4' (link https://huggingface.co/THUDM/glm-4-9b-chat-1m; claimed length 1M, effective length 64K), column '128K' = 83.1: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 128K column. Identity: catalogue glm-4-9b-chat-1m (HF zai-org/glm-4-9b-chat-1m).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-8b-prolong-512k-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.688,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-09-17",
    "note": "Open PT-LLM Leaderboard row 'princeton-nlp/Llama-3-8B-ProLong-512k-Instruct' (source=hub, precision=bfloat16, type=fine-tuned/fp on domain-specific datasets); published average 68.8 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha eae0626e85. Identity: catalogue llama-3-8b-prolong-512k-instruct (HF princeton-nlp/Llama-3-8B-ProLong-512k-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-8b-prolong-512k-instruct",
    "bench": "ruler-64k",
    "score": 0.832,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'ProLong' (link https://huggingface.co/princeton-nlp/Llama-3-8B-ProLong-512k-Instruct; claimed length 512K, effective length 32K), column '64K' = 83.2: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 512K covers the 64K column. Identity: catalogue llama-3-8b-prolong-512k-instruct (HF princeton-nlp/Llama-3-8B-ProLong-512k-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-8b-prolong-512k-instruct",
    "bench": "ruler-128k",
    "score": 0.816,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'ProLong' (link https://huggingface.co/princeton-nlp/Llama-3-8B-ProLong-512k-Instruct; claimed length 512K, effective length 32K), column '128K' = 81.6: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 512K covers the 128K column. Identity: catalogue llama-3-8b-prolong-512k-instruct (HF princeton-nlp/Llama-3-8B-ProLong-512k-Instruct).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "megabeam-mistral-7b-512k",
    "bench": "open-pt-llm-avg9",
    "score": 0.6258,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-10-07",
    "note": "Open PT-LLM Leaderboard row 'aws-prototyping/MegaBeam-Mistral-7B-512k' (source=hub, precision=bfloat16, type=fine-tuned/fp on domain-specific datasets); published average 62.58 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 3e3b8c4b93. Identity: catalogue megabeam-mistral-7b-512k (HF aws-prototyping/MegaBeam-Mistral-7B-512k).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "megabeam-mistral-7b-512k",
    "bench": "ruler-64k",
    "score": 0.837,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'MegaBeam-Mistral' (link https://huggingface.co/aws-prototyping/MegaBeam-Mistral-7B-512k; claimed length 512K, effective length 32K), column '64K' = 83.7: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 512K covers the 64K column. Identity: catalogue megabeam-mistral-7b-512k (HF aws-prototyping/MegaBeam-Mistral-7B-512k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "megabeam-mistral-7b-512k",
    "bench": "ruler-128k",
    "score": 0.837,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'MegaBeam-Mistral' (link https://huggingface.co/aws-prototyping/MegaBeam-Mistral-7B-512k; claimed length 512K, effective length 32K), column '128K' = 83.7: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 512K covers the 128K column. Identity: catalogue megabeam-mistral-7b-512k (HF aws-prototyping/MegaBeam-Mistral-7B-512k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-70b-instruct-gradient-1048k",
    "bench": "ruler-64k",
    "score": 0.809,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GradientAI/Llama3' (link https://huggingface.co/gradientai/Llama-3-70B-Instruct-Gradient-1048k; claimed length 1M, effective length 16K), column '64K' = 80.9: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 64K column. Identity: catalogue llama-3-70b-instruct-gradient-1048k (HF gradientai/Llama-3-70B-Instruct-Gradient-1048k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-70b-instruct-gradient-1048k",
    "bench": "ruler-128k",
    "score": 0.721,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GradientAI/Llama3' (link https://huggingface.co/gradientai/Llama-3-70B-Instruct-Gradient-1048k; claimed length 1M, effective length 16K), column '128K' = 72.1: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 128K column. Identity: catalogue llama-3-70b-instruct-gradient-1048k (HF gradientai/Llama-3-70B-Instruct-Gradient-1048k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-8b-instruct-gradient-1048k",
    "bench": "ruler-64k",
    "score": 0.763,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GradientAI/Llama3' (link https://huggingface.co/gradientai/Llama-3-8B-Instruct-Gradient-1048k; claimed length 1M, effective length 16K), column '64K' = 76.3: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 64K column. Identity: catalogue llama-3-8b-instruct-gradient-1048k (HF gradientai/Llama-3-8B-Instruct-Gradient-1048k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "llama-3-8b-instruct-gradient-1048k",
    "bench": "ruler-128k",
    "score": 0.695,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GradientAI/Llama3' (link https://huggingface.co/gradientai/Llama-3-8B-Instruct-Gradient-1048k; claimed length 1M, effective length 16K), column '128K' = 69.5: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 128K column. Identity: catalogue llama-3-8b-instruct-gradient-1048k (HF gradientai/Llama-3-8B-Instruct-Gradient-1048k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "yi-34b-200k",
    "bench": "open-pt-llm-avg9",
    "score": 0.6881,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-34B-200K' (source=hub, precision=bfloat16, type=pretrained); published average 68.81 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a443ece418. Identity: catalogue yi-34b-200k (HF 01-ai/Yi-34B-200K).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-34b-200k",
    "bench": "ruler-64k",
    "score": 0.832,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Yi' (link https://huggingface.co/01-ai/Yi-34B-200K; claimed length 200K, effective length 32K), column '64K' = 83.2: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 200K covers the 64K column. Identity: catalogue yi-34b-200k (HF 01-ai/Yi-34B-200K).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "yi-34b-200k",
    "bench": "ruler-128k",
    "score": 0.773,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'Yi' (link https://huggingface.co/01-ai/Yi-34B-200K; claimed length 200K, effective length 32K), column '128K' = 77.3: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 200K covers the 128K column. Identity: catalogue yi-34b-200k (HF 01-ai/Yi-34B-200K).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "internlm2-5-7b-chat-1m",
    "bench": "open-pt-llm-avg9",
    "score": 0.6927,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-05",
    "note": "Open PT-LLM Leaderboard row 'internlm/internlm2_5-7b-chat-1m' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 69.27 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 8d1a709a04. Identity: catalogue internlm2-5-7b-chat-1m (HF internlm/internlm2_5-7b-chat-1m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "internlm2-5-7b-chat-1m",
    "bench": "ruler-64k",
    "score": 0.755,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'InternLM2.5' (link https://huggingface.co/internlm/internlm2_5-7b-chat-1m; claimed length 1M, effective length 4K), column '64K' = 75.5: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 64K column. Identity: catalogue internlm2-5-7b-chat-1m (HF internlm/internlm2_5-7b-chat-1m).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "internlm2-5-7b-chat-1m",
    "bench": "ruler-128k",
    "score": 0.689,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'InternLM2.5' (link https://huggingface.co/internlm/internlm2_5-7b-chat-1m; claimed length 1M, effective length 4K), column '128K' = 68.9: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 128K column. Identity: catalogue internlm2-5-7b-chat-1m (HF internlm/internlm2_5-7b-chat-1m).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "chatglm3-6b-128k",
    "bench": "open-pt-llm-avg9",
    "score": 0.4898,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-05",
    "note": "Open PT-LLM Leaderboard row 'THUDM/chatglm3-6b-128k' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 48.98 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 38905c4c82. Identity: catalogue chatglm3-6b-128k (HF zai-org/chatglm3-6b-128k).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "chatglm3-6b-128k",
    "bench": "ruler-64k",
    "score": 0.56,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GLM3' (link https://huggingface.co/THUDM/chatglm3-6b-128K; claimed length 128K, effective length 4K), column '64K' = 56.0: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 128K covers the 64K column. Identity: catalogue chatglm3-6b-128k (HF zai-org/chatglm3-6b-128k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "chatglm3-6b-128k",
    "bench": "ruler-128k",
    "score": 0.42,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'GLM3' (link https://huggingface.co/THUDM/chatglm3-6b-128K; claimed length 128K, effective length 4K), column '128K' = 42.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 128K covers the 128K column. Identity: catalogue chatglm3-6b-128k (HF zai-org/chatglm3-6b-128k).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "lwm-text-chat-1m",
    "bench": "ruler-64k",
    "score": 0.681,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'LWM' (link https://huggingface.co/LargeWorldModel/LWM-Text-Chat-1M; claimed length 1M, effective length <4K), column '64K' = 68.1: mean over RULER's 13 synthetic tasks at 64K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 64K column. Identity: catalogue lwm-text-chat-1m (HF LargeWorldModel/LWM-Text-Chat-1M).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "lwm-text-chat-1m",
    "bench": "ruler-128k",
    "score": 0.65,
    "source": "https://github.com/NVIDIA/RULER/blob/c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a/README.md",
    "by": "independent",
    "date": "2025-10-09",
    "note": "NVIDIA RULER README (Apache-2.0) @c3f5e3b (README last changed 2025-10-09), main results table, row 'LWM' (link https://huggingface.co/LargeWorldModel/LWM-Text-Chat-1M; claimed length 1M, effective length <4K), column '128K' = 65.0: mean over RULER's 13 synthetic tasks at 128K tokens. README: 'We benchmark 17 open-source models across 4 task categories (in total 13 tasks) in RULER ... Here are our main results.' Rows reported by others are flagged in the README notes (asterisked Jamba-1.5-large, FILM-7B, Qwen2.5-1M, Qwen3, EXAONE 4.0) and are not used; this row has no asterisk, so it is NVIDIA's own run. Claimed context 1M covers the 128K column. Identity: catalogue lwm-text-chat-1m (HF LargeWorldModel/LWM-Text-Chat-1M).",
    "source_id": "nvidia-ruler-readme-results"
  },
  {
    "model": "qwen2-57b-a14b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7547,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-06-08",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen2-57B-A14B-Instruct' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 75.47 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 50896d66b3. Identity: catalogue qwen2-57b-a14b-instruct (HF Qwen/Qwen2-57B-A14B-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen2-57b-a14b-instruct",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.565314,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Qwen2-57B-A14B-Instruct' (query id Qwen/Qwen2-57B-A14B-Instruct, precision BF16, model sha 50896d66b39f1425d63720541a66c7df13e053c0); Average Accuracy = mean of the 21 published task scores = 56.5314 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue qwen2-57b-a14b-instruct (HF Qwen/Qwen2-57B-A14B-Instruct).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "hormoz-8b",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.530524,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Hormoz-8B' (query id mann-e/Hormoz-8B, precision F32, model sha c91bcecb236c90523f70db7efa23dd794e9b4cff); Average Accuracy = mean of the 21 published task scores = 53.0524 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue hormoz-8b (HF mann-e/Hormoz-8B).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "dorna2-llama3-1-8b-instruct",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.488086,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Dorna2-Llama3.1-8B-Instruct' (query id PartAI/Dorna2-Llama3.1-8B-Instruct, precision BF16, model sha b78e4bd261100c96e511ed5090ca0ce0e1f4b340); Average Accuracy = mean of the 21 published task scores = 48.8086 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue dorna2-llama3-1-8b-instruct (HF PartAI/Dorna2-Llama3.1-8B-Instruct).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "dorna-llama3-8b-instruct",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.470024,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Dorna-Llama3-8B-Instruct' (query id PartAI/Dorna-Llama3-8B-Instruct, precision BF16, model sha fb268bb51b950b4db5b7c82c1b73d9e803020eed); Average Accuracy = mean of the 21 published task scores = 47.0024 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue dorna-llama3-8b-instruct (HF PartAI/Dorna-Llama3-8B-Instruct).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "persianmind-v1-0",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.3714,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'PersianMind-v1.0' (query id universitytehran/PersianMind-v1.0, precision F32, model sha af603eeb074138e2a613fbc95d89f018afbd3041); Average Accuracy = mean of the 21 published task scores = 37.1400 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue persianmind-v1-0 (HF universitytehran/PersianMind-v1.0).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "maral-7b-alpha-1",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.333733,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'Maral-7B-alpha-1' (query id MaralGPT/Maral-7B-alpha-1, precision BF16, model sha 2ab5ca2a0d1a4454a78b4ca911e595bb9da2fe2f); Average Accuracy = mean of the 21 published task scores = 33.3733 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue maral-7b-alpha-1 (HF MaralGPT/Maral-7B-alpha-1).",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "bielik-11b-v2-0-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7257,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-10-17",
    "note": "Open PT-LLM Leaderboard row 'speakleash/Bielik-11B-v2.0-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 72.57 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha e4721e2af1. Identity: catalogue bielik-11b-v2-0-instruct (HF speakleash/Bielik-11B-v2.0-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bielik-11b-v2-0-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.652049,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.0-Instruct'; Space Average = mean(Sentiment 3.974, Language understanding 3.745, Phraseology 3.125, Tricky questions 2.197) = 3.2602 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-0-instruct (HF speakleash/Bielik-11B-v2.0-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-1-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7345,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-10-17",
    "note": "Open PT-LLM Leaderboard row 'speakleash/Bielik-11B-v2.1-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 73.45 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha c91776047e. Identity: catalogue bielik-11b-v2-1-instruct (HF speakleash/Bielik-11B-v2.1-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bielik-11b-v2-1-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.722352,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.1-Instruct'; Space Average = mean(Sentiment 3.955, Language understanding 3.915, Phraseology 3.105, Tricky questions 3.472) = 3.6118 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-1-instruct (HF speakleash/Bielik-11B-v2.1-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-2-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7345,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-10-17",
    "note": "Open PT-LLM Leaderboard row 'speakleash/Bielik-11B-v2.2-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 73.45 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 72a1ba0ea7. Identity: catalogue bielik-11b-v2-2-instruct (HF speakleash/Bielik-11B-v2.2-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bielik-11b-v2-2-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.691077,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.2-Instruct'; Space Average = mean(Sentiment 3.718, Language understanding 3.730, Phraseology 3.250, Tricky questions 3.124) = 3.4554 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-2-instruct (HF speakleash/Bielik-11B-v2.2-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-5-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.695326,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.5-Instruct'; Space Average = mean(Sentiment 4.006, Language understanding 3.860, Phraseology 3.130, Tricky questions 2.910) = 3.4766 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-5-instruct (HF speakleash/Bielik-11B-v2.5-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-6-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.727403,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.6-Instruct'; Space Average = mean(Sentiment 4.103, Language understanding 3.940, Phraseology 3.410, Tricky questions 3.096) = 3.6370 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-6-instruct (HF speakleash/Bielik-11B-v2.6-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v3-0-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.74693,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v3.0-Instruct'; Space Average = mean(Sentiment 3.878, Language understanding 3.910, Phraseology 3.965, Tricky questions 3.185) = 3.7346 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v3-0-instruct (HF speakleash/Bielik-11B-v3.0-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-4-5b-v3-0-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.675144,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-4.5B-v3.0-Instruct'; Space Average = mean(Sentiment 3.763, Language understanding 3.610, Phraseology 3.675, Tricky questions 2.455) = 3.3757 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-4-5b-v3-0-instruct (HF speakleash/Bielik-4.5B-v3.0-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-1-5b-v3-0-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.472737,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-1.5B-v3.0-Instruct'; Space Average = mean(Sentiment 3.526, Language understanding 2.330, Phraseology 2.380, Tricky questions 1.219) = 2.3637 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-1-5b-v3-0-instruct (HF speakleash/Bielik-1.5B-v3.0-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-minitron-7b-v3-0-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.675695,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-Minitron-7B-v3.0-Instruct'; Space Average = mean(Sentiment 3.718, Language understanding 3.830, Phraseology 3.230, Tricky questions 2.736) = 3.3785 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-minitron-7b-v3-0-instruct (HF speakleash/Bielik-Minitron-7B-v3.0-Instruct).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-7b-instruct-v0-1",
    "bench": "cptub-speakleash-avg",
    "score": 0.576852,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-7B-Instruct-v0.1'; Space Average = mean(Sentiment 3.590, Language understanding 3.475, Phraseology 2.315, Tricky questions 2.157) = 2.8843 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-7b-instruct-v0-1 (HF speakleash/Bielik-7B-Instruct-v0.1).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "llama-pllum-70b-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.705674,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/Llama-PLLuM-70B-chat'; Space Average = mean(Sentiment 3.940, Language understanding 3.610, Phraseology 3.350, Tricky questions 3.213) = 3.5284 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-pllum-70b-chat-2412 (HF CYFRAGOVPL/Llama-PLLuM-70B-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "llama-pllum-70b-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.665242,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/Llama-PLLuM-70B-instruct'; Space Average = mean(Sentiment 3.780, Language understanding 3.630, Phraseology 3.260, Tricky questions 2.635) = 3.3262 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-pllum-70b-instruct-2412 (HF CYFRAGOVPL/Llama-PLLuM-70B-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "llama-pllum-8b-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.58364,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/Llama-PLLuM-8B-chat'; Space Average = mean(Sentiment 3.130, Language understanding 2.930, Phraseology 3.360, Tricky questions 2.253) = 2.9182 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-pllum-8b-chat-2412 (HF CYFRAGOVPL/Llama-PLLuM-8B-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "llama-pllum-8b-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.563146,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/Llama-PLLuM-8B-instruct'; Space Average = mean(Sentiment 3.240, Language understanding 2.900, Phraseology 3.460, Tricky questions 1.663) = 2.8157 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue llama-pllum-8b-instruct-2412 (HF CYFRAGOVPL/Llama-PLLuM-8B-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.627494,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-12B-chat'; Space Average = mean(Sentiment 3.320, Language understanding 3.210, Phraseology 3.430, Tricky questions 2.590) = 3.1375 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-chat-2412 (HF CYFRAGOVPL/PLLuM-12B-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.618725,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-12B-instruct'; Space Average = mean(Sentiment 3.710, Language understanding 3.170, Phraseology 3.590, Tricky questions 1.904) = 3.0936 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-instruct-2412 (HF CYFRAGOVPL/PLLuM-12B-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-nc-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.63068,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-12B-nc-chat'; Space Average = mean(Sentiment 3.220, Language understanding 3.230, Phraseology 3.540, Tricky questions 2.624) = 3.1534 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-nc-chat-2412 (HF CYFRAGOVPL/PLLuM-12B-nc-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-nc-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.592657,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-12B-nc-instruct'; Space Average = mean(Sentiment 3.240, Language understanding 3.310, Phraseology 3.320, Tricky questions 1.983) = 2.9633 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-nc-instruct-2412 (HF CYFRAGOVPL/PLLuM-12B-nc-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-8x7b-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.601045,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-8x7B-chat'; Space Average = mean(Sentiment 3.440, Language understanding 3.450, Phraseology 3.350, Tricky questions 1.781) = 3.0052 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-8x7b-chat-2412 (HF CYFRAGOVPL/PLLuM-8x7B-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-8x7b-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.601281,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-8x7B-instruct'; Space Average = mean(Sentiment 3.590, Language understanding 3.470, Phraseology 3.460, Tricky questions 1.506) = 3.0064 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-8x7b-instruct-2412 (HF CYFRAGOVPL/PLLuM-8x7B-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-8x7b-nc-chat-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.605888,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-8x7B-nc-chat'; Space Average = mean(Sentiment 3.760, Language understanding 3.480, Phraseology 3.080, Tricky questions 1.798) = 3.0294 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-8x7b-nc-chat-2412 (HF CYFRAGOVPL/PLLuM-8x7B-nc-chat-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-8x7b-nc-instruct-2412",
    "bench": "cptub-speakleash-avg",
    "score": 0.622702,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/PLLuM-8x7B-nc-instruct'; Space Average = mean(Sentiment 3.880, Language understanding 3.590, Phraseology 3.220, Tricky questions 1.764) = 3.1135 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-8x7b-nc-instruct-2412 (HF CYFRAGOVPL/PLLuM-8x7B-nc-instruct-2412).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-nc-chat-2507",
    "bench": "cptub-speakleash-avg",
    "score": 0.733393,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/pllum-12b-nc-chat-250715'; Space Average = mean(Sentiment 4.359, Language understanding 3.955, Phraseology 3.455, Tricky questions 2.899) = 3.6670 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-nc-chat-2507 (HF CYFRAGOVPL/PLLuM-12b-nc-chat-2507).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "pllum-12b-nc-instruct-2507",
    "bench": "cptub-speakleash-avg",
    "score": 0.665052,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'CYFRAGOVPL/pllum-12b-nc-instruct-250715'; Space Average = mean(Sentiment 3.910, Language understanding 3.725, Phraseology 3.295, Tricky questions 2.371) = 3.3253 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue pllum-12b-nc-instruct-2507 (HF CYFRAGOVPL/PLLuM-12b-nc-instruct-2507).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "openchat-3-5-0106-gemma",
    "bench": "open-pt-llm-avg9",
    "score": 0.7267,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-17",
    "note": "Open PT-LLM Leaderboard row 'openchat/openchat-3.5-0106-gemma' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 72.67 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 5316738a90. Identity: catalogue openchat-3-5-0106-gemma (HF openchat/openchat-3.5-0106-gemma).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "openchat-3-5-0106-gemma",
    "bench": "cptub-speakleash-avg",
    "score": 0.546777,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'openchat/openchat-3.5-0106-gemma'; Space Average = mean(Sentiment 3.731, Language understanding 3.080, Phraseology 2.445, Tricky questions 1.680) = 2.7339 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue openchat-3-5-0106-gemma (HF openchat/openchat-3.5-0106-gemma).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "trurl-2-13b-academic",
    "bench": "cptub-speakleash-avg",
    "score": 0.461907,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'Voicelab/trurl-2-13b-academic'; Space Average = mean(Sentiment 3.301, Language understanding 2.755, Phraseology 2.165, Tricky questions 1.017) = 2.3095 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue trurl-2-13b-academic (HF Voicelab/trurl-2-13b-academic).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "h2o-danube2-1-8b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.4773,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-09",
    "note": "Open PT-LLM Leaderboard row 'h2oai/h2o-danube2-1.8b-chat' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 47.73 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 4000d05999. Identity: catalogue h2o-danube2-1-8b-chat (HF h2oai/h2o-danube2-1.8b-chat).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "h2o-danube2-1-8b-chat",
    "bench": "cptub-speakleash-avg",
    "score": 0.3283,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'h2oai/h2o-danube2-1.8b-chat'; Space Average = mean(Sentiment 2.372, Language understanding 1.595, Phraseology 2.470, Tricky questions 0.129) = 1.6415 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue h2o-danube2-1-8b-chat (HF h2oai/h2o-danube2-1.8b-chat).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "baichuan-m1-14b-instruct",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3608,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Baichuan-M1-14B-Instruct' (model link https://huggingface.co/baichuan-inc/Baichuan-M1-14B-Instruct); 'Average Performance' 36.08 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2025-01-23). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue baichuan-m1-14b-instruct (HF baichuan-inc/Baichuan-M1-14B-Instruct).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "baichuan-m2-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3802,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-09-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Baichuan-M2-32B' (model link https://huggingface.co/baichuan-inc/Baichuan-M2-32B); 'Average Performance' 38.02 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-09-04, model release 2025-08-10). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue baichuan-m2-32b (HF baichuan-inc/Baichuan-M2-32B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "k2-think",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3781,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-11-01",
    "note": "BRIDGE Zero-Shot leaderboard row 'K2-Think' (model link https://huggingface.co/LLM360/K2-Think); 'Average Performance' 37.81 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-11-01, model release 2025-09-08). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue k2-think (HF IFM/K2-Think).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "meditron-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.0952,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'meditron-7b' (model link https://github.com/epfLLM/meditron); 'Average Performance' 9.52 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2023-11-27). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue meditron-7b (HF epfl-llm/meditron-7b).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "meditron-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1568,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'meditron-70b' (model link https://github.com/epfLLM/meditron); 'Average Performance' 15.68 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2023-11-27). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue meditron-70b (HF epfl-llm/meditron-70b).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "llama3-openbiollm-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.142,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama3-OpenBioLLM-8B' (model link https://huggingface.co/aaditya/Llama3-OpenBioLLM-8B); 'Average Performance' 14.2 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-04-20). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue llama3-openbiollm-8b (HF aaditya/Llama3-OpenBioLLM-8B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "llama3-openbiollm-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3301,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama3-OpenBioLLM-70B' (model link https://huggingface.co/aaditya/Llama3-OpenBioLLM-70B); 'Average Performance' 33.01 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-04-24). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue llama3-openbiollm-70b (HF aaditya/Llama3-OpenBioLLM-70B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "mmed-llama-3-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2037,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'MMed-Llama-3-8B' (model link https://huggingface.co/Henrychur/MMed-Llama-3-8B); 'Average Performance' 20.37 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-05-24). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue mmed-llama-3-8b (HF Henrychur/MMed-Llama-3-8B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "llama-3-1-8b-ultramedical",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2016,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3.1-8B-UltraMedical' (model link https://huggingface.co/TsinghuaC3I/Llama-3.1-8B-UltraMedical); 'Average Performance' 20.16 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-09-01). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue llama-3-1-8b-ultramedical (HF TsinghuaC3I/Llama-3.1-8B-UltraMedical).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "llama-3-70b-ultramedical",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.334,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Llama-3-70B-UltraMedical' (model link https://huggingface.co/TsinghuaC3I/Llama-3-70B-UltraMedical); 'Average Performance' 33.4 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-09-02). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue llama-3-70b-ultramedical (HF TsinghuaC3I/Llama-3-70B-UltraMedical).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "biomistral-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2043,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'BioMistral-7B' (model link https://huggingface.co/BioMistral); 'Average Performance' 20.43 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-02-14). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue biomistral-7b (HF BioMistral/BioMistral-7B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "hulu-med-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3071,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2026-01-13",
    "note": "BRIDGE Zero-Shot leaderboard row 'Hulu-Med-7B' (model link https://huggingface.co/ZJU-AI4H/Hulu-Med-7B); 'Average Performance' 30.71 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2026-01-13, model release 2025-10-09). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue hulu-med-7b (HF ZJU-AI4H/Hulu-Med-7B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "hulu-med-14b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3545,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2026-01-13",
    "note": "BRIDGE Zero-Shot leaderboard row 'Hulu-Med-14B' (model link https://huggingface.co/ZJU-AI4H/Hulu-Med-14B); 'Average Performance' 35.45 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2026-01-13, model release 2025-10-09). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue hulu-med-14b (HF ZJU-AI4H/Hulu-Med-14B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "hulu-med-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3829,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2026-01-13",
    "note": "BRIDGE Zero-Shot leaderboard row 'Hulu-Med-32B' (model link https://huggingface.co/ZJU-AI4H/Hulu-Med-32B); 'Average Performance' 38.29 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2026-01-13, model release 2025-10-09). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue hulu-med-32b (HF ZJU-AI4H/Hulu-Med-32B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "biomni-r0-32b-preview",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3534,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-11-04",
    "note": "BRIDGE Zero-Shot leaderboard row 'Biomni-R0-32B-Preview-Thinking' (model link https://huggingface.co/biomni/Biomni-R0-32B-Preview); 'Average Performance' 35.34 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-11-04, model release 2025-10-06). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue biomni-r0-32b-preview (HF biomni/Biomni-R0-32B-Preview).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "antangelmed",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3665,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2026-01-13",
    "note": "BRIDGE Zero-Shot leaderboard row 'AntAngelMed' (model link https://huggingface.co/MedAIBase/AntAngelMed); 'Average Performance' 36.65 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2026-01-13, model release 2025-12-12). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue antangelmed (HF MedAIBase/AntAngelMed).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "qwenlong-l1-32b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3925,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'QwenLong-L1-32B' (model link https://huggingface.co/Tongyi-Zhiwen/QwenLong-L1-32B); 'Average Performance' 39.25 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2025-05-23). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue qwenlong-l1-32b (HF Tongyi-Zhiwen/QwenLong-L1-32B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "yi-1-5-9b-chat-16k",
    "bench": "open-pt-llm-avg9",
    "score": 0.6785,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-9B-Chat-16K' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 67.85 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 867b600699. Identity: catalogue yi-1-5-9b-chat-16k (HF 01-ai/Yi-1.5-9B-Chat-16K).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-1-5-9b-chat-16k",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2881,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Yi-1.5-9B-Chat-16K' (model link https://huggingface.co/01-ai/Yi-1.5-9B-Chat-16K); 'Average Performance' 28.81 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-05-15). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue yi-1-5-9b-chat-16k (HF 01-ai/Yi-1.5-9B-Chat-16K).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "yi-1-5-34b-chat-16k",
    "bench": "open-pt-llm-avg9",
    "score": 0.7194,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open PT-LLM Leaderboard row '01-ai/Yi-1.5-34B-Chat-16K' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 71.94 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 12fa84d108. Identity: catalogue yi-1-5-34b-chat-16k (HF 01-ai/Yi-1.5-34B-Chat-16K).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "yi-1-5-34b-chat-16k",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.3212,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-04-28",
    "note": "BRIDGE Zero-Shot leaderboard row 'Yi-1.5-34B-Chat-16K' (model link https://huggingface.co/01-ai/Yi-1.5-34B-Chat-16K); 'Average Performance' 32.12 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-04-28, model release 2024-05-15). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue yi-1-5-34b-chat-16k (HF 01-ai/Yi-1.5-34B-Chat-16K).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "huatuogpt-o1-7b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2959,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'HuatuoGPT-o1-7B' (model link https://huggingface.co/FreedomIntelligence/HuatuoGPT-o1-7B); 'Average Performance' 29.59 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2025-01-06). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue huatuogpt-o1-7b (HF FreedomIntelligence/HuatuoGPT-o1-7B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "huatuogpt-o1-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2586,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'HuatuoGPT-o1-8B' (model link https://huggingface.co/FreedomIntelligence/HuatuoGPT-o1-8B); 'Average Performance' 25.86 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2024-12-27). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue huatuogpt-o1-8b (HF FreedomIntelligence/HuatuoGPT-o1-8B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "huatuogpt-o1-70b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.339,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'HuatuoGPT-o1-70B' (model link https://huggingface.co/FreedomIntelligence/HuatuoGPT-o1-70B); 'Average Performance' 33.9 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2024-12-27). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue huatuogpt-o1-70b (HF FreedomIntelligence/HuatuoGPT-o1-70B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "huatuogpt-o1-72b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4101,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'HuatuoGPT-o1-72B' (model link https://huggingface.co/FreedomIntelligence/HuatuoGPT-o1-72B); 'Average Performance' 41.01 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2025-01-29). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue huatuogpt-o1-72b (HF FreedomIntelligence/HuatuoGPT-o1-72B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "medreason-8b",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.1817,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-07-22",
    "note": "BRIDGE Zero-Shot leaderboard row 'MedReason-8B' (model link https://huggingface.co/UCSC-VLAA/MedReason-8B); 'Average Performance' 18.17 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md: added 2025-07-22, model release 2025-04-01). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38. Identity: catalogue medreason-8b (HF UCSC-VLAA/MedReason-8B).",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "falcon-h1-34b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7861,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-08",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon-H1-34B-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 78.61 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 6e6890e26c. Identity: catalogue falcon-h1-34b-instruct (HF tiiuae/Falcon-H1-34B-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "falcon-h1-34b-instruct",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5361,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'tiiuae/Falcon-H1-34B-Instruct'; 3C3H Score 0.5361 (95% CI 0.484587-0.588859, n=264); judge claude-3.5-sonnet; 264/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue falcon-h1-34b-instruct (HF tiiuae/Falcon-H1-34B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-34b-instruct",
    "bench": "aragen-v3-3c3h",
    "score": 0.4574,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'tiiuae/Falcon-H1-34B-Instruct'; 3C3H Score 0.4574 (95% CI 0.410149-0.505315, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_051309; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue falcon-h1-34b-instruct (HF tiiuae/Falcon-H1-34B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-7b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7534,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-08",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon-H1-7B-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 75.34 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 179c218c36. Identity: catalogue falcon-h1-7b-instruct (HF tiiuae/Falcon-H1-7B-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "falcon-h1-7b-instruct",
    "bench": "hindigen-v1-3c3h",
    "score": 0.3497,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'tiiuae/Falcon-H1-7B-Instruct'; 3C3H Score 0.3497 (95% CI 0.302575-0.396878, n=261); judge claude-3.5-sonnet; 261/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue falcon-h1-7b-instruct (HF tiiuae/Falcon-H1-7B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-7b-instruct",
    "bench": "aragen-v3-3c3h",
    "score": 0.244,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'tiiuae/Falcon-H1-7B-Instruct'; 3C3H Score 0.244 (95% CI 0.204472-0.285365, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_081044; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue falcon-h1-7b-instruct (HF tiiuae/Falcon-H1-7B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-3b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7139,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-08",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon-H1-3B-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 71.39 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a3fcfd293e. Identity: catalogue falcon-h1-3b-instruct (HF tiiuae/Falcon-H1-3B-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "falcon-h1-3b-instruct",
    "bench": "hindigen-v1-3c3h",
    "score": 0.1444,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'tiiuae/Falcon-H1-3B-Instruct'; 3C3H Score 0.1444 (95% CI 0.109783-0.180343, n=262); judge claude-3.5-sonnet; 262/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue falcon-h1-3b-instruct (HF tiiuae/Falcon-H1-3B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-3b-instruct",
    "bench": "aragen-v3-3c3h",
    "score": 0.1505,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'tiiuae/Falcon-H1-3B-Instruct'; 3C3H Score 0.1505 (95% CI 0.118356-0.18343, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_105754; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue falcon-h1-3b-instruct (HF tiiuae/Falcon-H1-3B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-1-5b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.6301,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-08",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/Falcon-H1-1.5B-Instruct' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 63.01 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 1d5e1e53ea. Identity: catalogue falcon-h1-1-5b-instruct (HF tiiuae/Falcon-H1-1.5B-Instruct).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "falcon-h1-1-5b-instruct",
    "bench": "hindigen-v1-3c3h",
    "score": 0.0635,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'tiiuae/Falcon-H1-1.5B-Instruct'; 3C3H Score 0.0635 (95% CI 0.038149-0.090995, n=261); judge claude-3.5-sonnet; 261/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue falcon-h1-1-5b-instruct (HF tiiuae/Falcon-H1-1.5B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "falcon-h1-1-5b-instruct",
    "bench": "aragen-v3-3c3h",
    "score": 0.0458,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'tiiuae/Falcon-H1-1.5B-Instruct'; 3C3H Score 0.0458 (95% CI 0.027049-0.065888, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_134905; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue falcon-h1-1-5b-instruct (HF tiiuae/Falcon-H1-1.5B-Instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-590m-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.067,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-590m-chat'; 'Ar Prompt-lvl' 0.067 (Ar instruction-level 0.109, En prompt-level 0.143 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-590m-chat (HF inception42/jais-family-590m-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-1p3b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.136,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-1p3b-chat'; 'Ar Prompt-lvl' 0.136 (Ar instruction-level 0.234, En prompt-level 0.149 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-1p3b-chat (HF inception42/jais-family-1p3b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-2p7b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.171,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-2p7b-chat'; 'Ar Prompt-lvl' 0.171 (Ar instruction-level 0.264, En prompt-level 0.194 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-2p7b-chat (HF inception42/jais-family-2p7b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-6p7b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.225,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-6p7b-chat'; 'Ar Prompt-lvl' 0.225 (Ar instruction-level 0.321, En prompt-level 0.267 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-6p7b-chat (HF inception42/jais-family-6p7b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-6p7b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.2129,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-09-29",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/jais-family-6p7b-chat'; 3C3H Score 0.2129 (95% CI 0.174939-0.253001, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250929_005659; precision float32; RM_Thinking_Traces=None. Identity: catalogue jais-family-6p7b-chat (HF inception42/jais-family-6p7b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-13b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.285,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-13b-chat'; 'Ar Prompt-lvl' 0.285 (Ar instruction-level 0.375, En prompt-level 0.312 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-13b-chat (HF inception42/jais-family-13b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-13b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.2414,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/jais-family-13b-chat'; 3C3H Score 0.2414 (95% CI 0.201432-0.28271, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_192902; precision float32; RM_Thinking_Traces=None. Identity: catalogue jais-family-13b-chat (HF inception42/jais-family-13b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-30b-8k-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.285,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-30b-8k-chat'; 'Ar Prompt-lvl' 0.285 (Ar instruction-level 0.386, En prompt-level 0.357 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-30b-8k-chat (HF inception42/jais-family-30b-8k-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-30b-8k-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.3077,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/jais-family-30b-8k-chat'; 3C3H Score 0.3077 (95% CI 0.264614-0.351897, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_223729; precision float32; RM_Thinking_Traces=None. Identity: catalogue jais-family-30b-8k-chat (HF inception42/jais-family-30b-8k-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-30b-16k-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.29,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-family-30b-16k-chat'; 'Ar Prompt-lvl' 0.29 (Ar instruction-level 0.403, En prompt-level 0.361 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-family-30b-16k-chat (HF inception42/jais-family-30b-16k-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-family-30b-16k-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.3233,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/jais-family-30b-16k-chat'; 3C3H Score 0.3233 (95% CI 0.279536-0.367458, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_170053; precision float32; RM_Thinking_Traces=None. Identity: catalogue jais-family-30b-16k-chat (HF inception42/jais-family-30b-16k-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-adapted-7b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.225,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-adapted-7b-chat'; 'Ar Prompt-lvl' 0.225 (Ar instruction-level 0.339, En prompt-level 0.371 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-adapted-7b-chat (HF inception42/jais-adapted-7b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-adapted-13b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.344,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-adapted-13b-chat'; 'Ar Prompt-lvl' 0.344 (Ar instruction-level 0.462, En prompt-level 0.447 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-adapted-13b-chat (HF inception42/jais-adapted-13b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-adapted-70b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.322,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'inceptionai/jais-adapted-70b-chat'; 'Ar Prompt-lvl' 0.322 (Ar instruction-level 0.418, En prompt-level 0.443 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue jais-adapted-70b-chat (HF inception42/jais-adapted-70b-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-2-70b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.5239,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-12-14",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/Jais-2-70B-Chat'; 3C3H Score 0.5239 (95% CI 0.477922-0.569362, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251214_195133; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue jais-2-70b-chat (HF inception42/Jais-2-70B-Chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "jais-2-8b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.3395,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "lab",
    "date": "2025-12-14",
    "note": "Inception AraGen-v3 leaderboard entry 'inceptionai/Jais-2-8B-Chat'; 3C3H Score 0.3395 (95% CI 0.296298-0.383621, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20251214_220746; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue jais-2-8b-chat (HF inception42/Jais-2-8B-Chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "acegpt-13b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.327,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'FreedomIntelligence/AceGPT-13B-chat'; 'Ar Prompt-lvl' 0.327 (Ar instruction-level 0.443, En prompt-level 0.406 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue acegpt-13b-chat (HF FreedomIntelligence/AceGPT-13B-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "acegpt-13b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.1591,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'FreedomIntelligence/AceGPT-13B-chat'; 3C3H Score 0.1591 (95% CI 0.124139-0.194961, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20250930_090751; precision float16; RM_Thinking_Traces=None. Identity: catalogue acegpt-13b-chat (HF FreedomIntelligence/AceGPT-13B-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "acegpt-7b-chat",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.22,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'FreedomIntelligence/AceGPT-7B-chat'; 'Ar Prompt-lvl' 0.22 (Ar instruction-level 0.321, En prompt-level 0.273 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue acegpt-7b-chat (HF FreedomIntelligence/AceGPT-7B-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "acegpt-7b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.1245,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'FreedomIntelligence/AceGPT-7B-chat'; 3C3H Score 0.1245 (95% CI 0.09417-0.156521, n=323); judge claude-sonnet-4-20250514; 323/324 entries answered; run 20250930_130644; precision float16; RM_Thinking_Traces=None. Identity: catalogue acegpt-7b-chat (HF FreedomIntelligence/AceGPT-7B-chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "acegpt-v2-8b-chat",
    "bench": "aragen-v3-3c3h",
    "score": 0.3503,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'FreedomIntelligence/AceGPT-v2-8B-Chat'; 3C3H Score 0.3503 (95% CI 0.304642-0.397269, n=320); judge claude-sonnet-4-20250514; 320/324 entries answered; run 20250930_053254; precision float16; RM_Thinking_Traces=None. Identity: catalogue acegpt-v2-8b-chat (HF FreedomIntelligence/AceGPT-v2-8B-Chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "silma-9b-instruct-v1-0",
    "bench": "arabic-ifeval-inception-ar-prompt",
    "score": 0.342,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception Arabic IFEval entry 'silma-ai/SILMA-9B-Instruct-v1.0'; 'Ar Prompt-lvl' 0.342 (Ar instruction-level 0.432, En prompt-level 0.525 not imported). No run date in the file; date is the pinned Space revision date. Identity: catalogue silma-9b-instruct-v1-0 (HF silma-ai/SILMA-9B-Instruct-v1.0).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "silma-9b-instruct-v1-0",
    "bench": "aragen-v3-3c3h",
    "score": 0.067,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-10-01",
    "note": "Inception AraGen-v3 leaderboard entry 'silma-ai/SILMA-9B-Instruct-v1.0'; 3C3H Score 0.067 (95% CI 0.042334-0.094283, n=312); judge claude-sonnet-4-20250514; 312/324 entries answered; run 20251001_011635; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue silma-9b-instruct-v1-0 (HF silma-ai/SILMA-9B-Instruct-v1.0).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "calme-2-1-qwen2-5-72b",
    "bench": "aragen-v3-3c3h",
    "score": 0.5081,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-28",
    "note": "Inception AraGen-v3 leaderboard entry 'MaziyarPanahi/calme-2.1-qwen2.5-72b'; 3C3H Score 0.5081 (95% CI 0.461894-0.554019, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250928_015033; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue calme-2-1-qwen2-5-72b (HF MaziyarPanahi/calme-2.1-qwen2.5-72b).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "calme-2-2-qwen2-5-72b",
    "bench": "aragen-v3-3c3h",
    "score": 0.5031,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-27",
    "note": "Inception AraGen-v3 leaderboard entry 'MaziyarPanahi/calme-2.2-qwen2.5-72b'; 3C3H Score 0.5031 (95% CI 0.457436-0.54926, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250927_200200; precision bfloat16; RM_Thinking_Traces=None. Identity: catalogue calme-2-2-qwen2-5-72b (HF MaziyarPanahi/calme-2.2-qwen2.5-72b).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "sarvam-m",
    "bench": "hindigen-v1-3c3h",
    "score": 0.2292,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'sarvamai/sarvam-m'; 3C3H Score 0.2292 (95% CI 0.18967-0.269131, n=265); judge claude-3.5-sonnet; 265/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue sarvam-m (HF sarvamai/sarvam-m).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "airavata",
    "bench": "hindigen-v1-3c3h",
    "score": 0.0957,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'ai4bharat/Airavata'; 3C3H Score 0.0957 (95% CI 0.064742-0.129809, n=260); judge claude-3.5-sonnet; 260/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue airavata (HF ai4bharat/Airavata).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "llama-3-nanda-10b-chat",
    "bench": "hindigen-v1-3c3h",
    "score": 0.3173,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "lab",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'MBZUAI/Llama-3-Nanda-10B-Chat'; 3C3H Score 0.3173 (95% CI 0.273095-0.362457, n=258); judge claude-3.5-sonnet; 258/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue llama-3-nanda-10b-chat (HF MBZUAI/Llama-3-Nanda-10B-Chat).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "krutrim-2-instruct",
    "bench": "hindigen-v1-3c3h",
    "score": 0.5962,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'krutrim-ai-labs/Krutrim-2-instruct'; 3C3H Score 0.5962 (95% CI 0.541126-0.651117, n=260); judge claude-3-7-sonnet-20250219; 260/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue krutrim-2-instruct (HF krutrim-ai-labs/Krutrim-2-instruct).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "orca-2-13b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.975,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Microsoft Orca-2-13b': factual consistency rate 97.5% (hallucination rate 2.5%), answer rate 100.0%, average summary length 66.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue orca-2-13b (HF microsoft/Orca-2-13b).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "neural-chat-7b-v3-3",
    "bench": "open-pt-llm-avg9",
    "score": 0.6707,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'Intel/neural-chat-7b-v3-3' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 67.07 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha cdce282d00. Identity: catalogue neural-chat-7b-v3-3 (HF Intel/neural-chat-7b-v3-3).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "neural-chat-7b-v3-3",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.974,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Intel Neural-Chat-7B-v3-3': factual consistency rate 97.4% (hallucination rate 2.6%), answer rate 100.0%, average summary length 60.7 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue neural-chat-7b-v3-3 (HF Intel/neural-chat-7b-v3-3).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "internlm3-8b-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.96,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'InternLM3-8B-Instruct': factual consistency rate 96.0% (hallucination rate 4.0%), answer rate 100.0%, average summary length 97.5 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue internlm3-8b-instruct (HF internlm/internlm3-8b-instruct).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "openelm-3b-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.752,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Apple OpenELM-3B-Instruct': factual consistency rate 75.2% (hallucination rate 24.8%), answer rate 99.3%, average summary length 47.2 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue openelm-3b-instruct (HF apple/OpenELM-3B-Instruct).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "deepseek-tng-r1t2-chimera",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.945,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'TNGTech DeepSeek-TNG-R1T2-Chimera': factual consistency rate 94.5% (hallucination rate 5.5%), answer rate 99.6%, average summary length 78.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue deepseek-tng-r1t2-chimera (HF tngtech/DeepSeek-TNG-R1T2-Chimera).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "jamba-1-6-large",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.977,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'AI21 Jamba-1.6-Large': factual consistency rate 97.7% (hallucination rate 2.3%), answer rate 99.9%, average summary length 85.6 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue jamba-1-6-large (HF ai21labs/AI21-Jamba-Large-1.6).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "jamba-1-6-mini",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.954,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'AI21 Jamba-1.6-Mini': factual consistency rate 95.4% (hallucination rate 4.6%), answer rate 100.0%, average summary length 82.3 words. HHEM-2.3 judge; the model identity follows the README's API integration details. Identity: catalogue jamba-1-6-mini (HF ai21labs/AI21-Jamba-Mini-1.6).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "gemma-2-9b-it-wpo-hb",
    "bench": "alpacaeval2-lc",
    "score": 0.7673,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gemma-2-9b-it-WPO-HB' (mode 'community'), length_controlled_winrate = 76.73 (LC standard error 0.42); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 77.83; average length 2285. Config models_configs/gemma-2-9b-it-WPO-HB/configs.yaml: model_name 'wzhouad/gemma-2-9b-it-WPO-HB', fn_completions 'vllm_local_completions'. Identity: catalogue gemma-2-9b-it-wpo-hb (HF wzhouad/gemma-2-9b-it-WPO-HB).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama3-instruct-8b-wpo-hb-v2",
    "bench": "alpacaeval2-lc",
    "score": 0.5337,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama-3-Instruct-8B-WPO-HB-v2' (mode 'community'), length_controlled_winrate = 53.37 (LC standard error 0.71); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 57.33; average length 2472. Config models_configs/Llama-3-Instruct-8B-WPO-HB-v2/configs.yaml: model_name 'wzhouad/Llama3-Instruct-8B-WPO-HB-v2', fn_completions 'vllm_local_completions'. Identity: catalogue llama3-instruct-8b-wpo-hb-v2 (HF wzhouad/Llama3-Instruct-8B-WPO-HB-v2).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fusechat-gemma-2-9b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.7018,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FuseChat-Gemma-2-9B-Instruct' (mode 'community'), length_controlled_winrate = 70.18 (LC standard error 0.59); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 70.50; average length 2155. Config models_configs/FuseChat-Gemma-2-9B-Instruct/configs.yaml: model_name 'FuseAI/FuseChat-Gemma-2-9B-Instruct', fn_completions 'vllm_local_completions'. Identity: catalogue fusechat-gemma-2-9b-instruct (HF FuseAI/FuseChat-Gemma-2-9B-Instruct).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fusechat-llama-3-1-8b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.6539,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FuseChat-Llama-3.1-8B-Instruct' (mode 'community'), length_controlled_winrate = 65.39 (LC standard error 0.67); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 63.33; average length 2033. Config models_configs/FuseChat-Llama-3.1-8B-Instruct/configs.yaml: model_name 'FuseAI/FuseChat-Llama-3.1-8B-Instruct', fn_completions 'vllm_local_completions'. Identity: catalogue fusechat-llama-3-1-8b-instruct (HF FuseAI/FuseChat-Llama-3.1-8B-Instruct).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fusechat-qwen-2-5-7b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.6358,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FuseChat-Qwen-2.5-7B-Instruct' (mode 'community'), length_controlled_winrate = 63.58 (LC standard error 0.62); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 64.64; average length 2173. Config models_configs/FuseChat-Qwen-2.5-7B-Instruct/configs.yaml: model_name 'FuseAI/FuseChat-Qwen-2.5-7B-Instruct', fn_completions 'vllm_local_completions'. Identity: catalogue fusechat-qwen-2-5-7b-instruct (HF FuseAI/FuseChat-Qwen-2.5-7B-Instruct).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fusechat-llama-3-2-3b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.54,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FuseChat-Llama-3.2-3B-Instruct' (mode 'community'), length_controlled_winrate = 54.00 (LC standard error 0.69); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 51.30; average length 1976. Config models_configs/FuseChat-Llama-3.2-3B-Instruct/configs.yaml: model_name 'FuseAI/FuseChat-Llama-3.2-3B-Instruct', fn_completions 'vllm_local_completions'. Identity: catalogue fusechat-llama-3-2-3b-instruct (HF FuseAI/FuseChat-Llama-3.2-3B-Instruct).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fusechat-llama-3-2-1b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.2527,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FuseChat-Llama-3.2-1B-Instruct' (mode 'community'), length_controlled_winrate = 25.27 (LC standard error 0.64); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 29.92; average length 2259. Config models_configs/FuseChat-Llama-3.2-1B-Instruct/configs.yaml: model_name 'FuseAI/FuseChat-Llama-3.2-1B-Instruct', fn_completions 'vllm_local_completions'. Identity: catalogue fusechat-llama-3-2-1b-instruct (HF FuseAI/FuseChat-Llama-3.2-1B-Instruct).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama3-pbm-nova-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.6239,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama3-PBM-Nova-70B' (mode 'community'), length_controlled_winrate = 62.39 (LC standard error 0.76); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 62.95; average length 2207. Config models_configs/Llama3-PBM-Nova-70B/configs.yaml: model_name 'PKU-Baichuan-MLSystemLab/Llama3-PBM-Nova-70B', fn_completions 'openai_completions'. Identity: catalogue llama3-pbm-nova-70b (HF PKU-Baichuan-MLSystemLab/Llama3-PBM-Nova-70B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "storm-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.5045,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Storm-7B' (mode 'community'), length_controlled_winrate = 50.45 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 50.27; average length 2045. Config models_configs/Storm-7B/configs.yaml: model_name 'jieliu/Storm-7B', fn_completions 'vllm_local_completions'. Identity: catalogue storm-7b (HF jieliu/Storm-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gemma-2-9b-it-sppo-iter3",
    "bench": "alpacaeval2-lc",
    "score": 0.5397,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'SPPO-Gemma-2-9B-It-PairRM' (mode 'community'), length_controlled_winrate = 53.97 (LC standard error 0.65); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 48.23; average length 1803. Config models_configs/SPPO-Gemma-2-9B-It-PairRM/configs.yaml: model_name 'UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3', fn_completions 'vllm_local_completions'. Identity: catalogue gemma-2-9b-it-sppo-iter3 (HF UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gemma-2-9b-it-sppo-iter3",
    "bench": "open-pt-llm-avg9",
    "score": 0.743,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-07",
    "note": "Open PT-LLM Leaderboard row 'UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 74.3 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 2261f2a03b. Identity: catalogue gemma-2-9b-it-sppo-iter3 (HF UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-instruct-8b-sppo-iter3",
    "bench": "alpacaeval2-lc",
    "score": 0.3856,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'SPPO-Llama-3-Instruct-8B-PairRM' (mode 'community'), length_controlled_winrate = 38.56 (LC standard error 0.87); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 39.67; average length 2066. Config models_configs/SPPO-Llama-3-Instruct-8B-PairRM/configs.yaml: model_name 'UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3', fn_completions 'vllm_local_completions'. Identity: catalogue llama-3-instruct-8b-sppo-iter3 (HF UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "llama-3-instruct-8b-sppo-iter3",
    "bench": "open-pt-llm-avg9",
    "score": 0.6914,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-07",
    "note": "Open PT-LLM Leaderboard row 'UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 69.14 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 48c29bf2d9. Identity: catalogue llama-3-instruct-8b-sppo-iter3 (HF UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "mistral7b-pairrm-sppo",
    "bench": "alpacaeval2-lc",
    "score": 0.3049,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'SPPO-Mistral7B-PairRM' (mode 'community'), length_controlled_winrate = 30.49 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 32.25; average length 2114. Config models_configs/SPPO-Mistral7B-PairRM/configs.yaml: model_name 'UCLA-AGI/Mistral7B-PairRM-SPPO', fn_completions 'vllm_local_completions'. Identity: catalogue mistral7b-pairrm-sppo (HF UCLA-AGI/Mistral7B-PairRM-SPPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral7b-pairrm-sppo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6496,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-09-21",
    "note": "Open PT-LLM Leaderboard row 'UCLA-AGI/Mistral7B-PairRM-SPPO' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 64.96 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha abdc173603. Identity: catalogue mistral7b-pairrm-sppo (HF UCLA-AGI/Mistral7B-PairRM-SPPO).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-8b-instruct-rainbowpo",
    "bench": "alpacaeval2-lc",
    "score": 0.5166,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama-3-Instruct-8B-RainbowPO' (mode 'community'), length_controlled_winrate = 51.66 (LC standard error 0.78); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 47.92; average length 1878. Config models_configs/Llama-3-Instruct-8B-RainbowPO/configs.yaml: model_name 'BraceZHY/Llama-3-8B-Instruct-RainbowPO', fn_completions 'vllm_local_completions'. Identity: catalogue llama-3-8b-instruct-rainbowpo (HF BraceZHY/Llama-3-8B-Instruct-RainbowPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "rebel-llama-3-armo-iter-1",
    "bench": "alpacaeval2-lc",
    "score": 0.4931,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'REBEL-Llama-3-8B-Instruct-Armo' (mode 'community'), length_controlled_winrate = 49.31 (LC standard error 0.71); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 48.44; average length 1965. Config models_configs/REBEL-Llama-3-8B-Instruct-Armo/configs.yaml: model_name 'Cornell-AGI/REBEL-Llama-3-Armo-iter_1', fn_completions 'vllm_local_completions'. Identity: catalogue rebel-llama-3-armo-iter-1 (HF Cornell-AGI/REBEL-Llama-3-Armo-iter_1).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "rebel-llama-3",
    "bench": "alpacaeval2-lc",
    "score": 0.314,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'REBEL-Llama-3-8B-Instruct' (mode 'community'), length_controlled_winrate = 31.40 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 34.31; average length 2372. Config models_configs/REBEL-Llama-3-8B-Instruct/configs.yaml: model_name 'Cornell-AGI/REBEL-Llama-3', fn_completions 'huggingface_local_completions'. Identity: catalogue rebel-llama-3 (HF Cornell-AGI/REBEL-Llama-3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-7m-gen-llama3-1-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.461,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-7M-Gen-Llama3_1-70B' (mode 'community'), length_controlled_winrate = 46.10 (LC standard error 0.82); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 37.46; average length 1654. Config models_configs/Infinity-Instruct-7M-Gen-Llama3_1-70B/configs.yaml: model_name 'baai/Infinity-Instruct-7M-Gen-Llama3_1-70B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-7m-gen-llama3-1-70b (HF BAAI/Infinity-Instruct-7M-Gen-Llama3_1-70B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-7m-gen-llama3-1-8b",
    "bench": "alpacaeval2-lc",
    "score": 0.3392,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-7M-Gen-Llama3_1-8B' (mode 'community'), length_controlled_winrate = 33.92 (LC standard error 0.85); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 28.53; average length 1640. Config models_configs/Infinity-Instruct-7M-Gen-Llama3_1-8B/configs.yaml: model_name 'baai/Infinity-Instruct-7M-Gen-Llama3_1-8B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-7m-gen-llama3-1-8b (HF BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-7m-gen-llama3-1-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.716,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-22",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B' (source=hub, precision=bfloat16, type=fine-tuned/fp on domain-specific datasets); published average 71.6 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 56f9c2845a. Identity: catalogue infinity-instruct-7m-gen-llama3-1-8b (HF BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "infinity-instruct-7m-gen-mistral-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.3967,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-7M-Gen-mistral-7B' (mode 'community'), length_controlled_winrate = 39.67 (LC standard error 0.80); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 34.35; average length 1742. Config models_configs/Infinity-Instruct-7M-Gen-mistral-7B/configs.yaml: model_name 'baai/Infinity-Instruct-7M-Gen-mistral-7B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-7m-gen-mistral-7b (HF BAAI/Infinity-Instruct-7M-Gen-mistral-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-llama3-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.3798,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0625-Llama3-70B' (mode 'community'), length_controlled_winrate = 37.98 (LC standard error 0.82); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 24.28; average length 1294. Config models_configs/Infinity-Instruct-3M-0625-Llama3-70B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0625-Llama3-70B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0625-llama3-70b (HF BAAI/Infinity-Instruct-3M-0625-Llama3-70B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-llama3-8b",
    "bench": "alpacaeval2-lc",
    "score": 0.2752,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0625-Llama3-8B' (mode 'community'), length_controlled_winrate = 27.52 (LC standard error 0.76); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 19.36; average length 1336. Config models_configs/Infinity-Instruct-3M-0625-Llama3-8B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0625-Llama3-8B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0625-llama3-8b (HF BAAI/Infinity-Instruct-3M-0625-Llama3-8B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-llama3-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6905,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-18",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-3M-0625-Llama3-8B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 69.05 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 7be7c0ff1e. Identity: catalogue infinity-instruct-3m-0625-llama3-8b (HF BAAI/Infinity-Instruct-3M-0625-Llama3-8B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "infinity-instruct-3m-0625-mistral-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.3142,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0625-Mistral-7B' (mode 'community'), length_controlled_winrate = 31.42 (LC standard error 0.80); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 21.09; average length 1305. Config models_configs/Infinity-Instruct-3M-0625-Mistral-7B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0625-Mistral-7B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0625-mistral-7b (HF BAAI/Infinity-Instruct-3M-0625-Mistral-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-mistral-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7091,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-18",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-3M-0625-Mistral-7B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.91 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 302e3ae0bc. Identity: catalogue infinity-instruct-3m-0625-mistral-7b (HF BAAI/Infinity-Instruct-3M-0625-Mistral-7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "infinity-instruct-3m-0625-qwen2-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.2187,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0625-Qwen2-7B' (mode 'community'), length_controlled_winrate = 21.87 (LC standard error 0.70); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.32; average length 1315. Config models_configs/Infinity-Instruct-3M-0625-Qwen2-7B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0625-Qwen2-7B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0625-qwen2-7b (HF BAAI/Infinity-Instruct-3M-0625-Qwen2-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-qwen2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7096,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-16",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-3M-0625-Qwen2-7B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.96 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 503c24156d. Identity: catalogue infinity-instruct-3m-0625-qwen2-7b (HF BAAI/Infinity-Instruct-3M-0625-Qwen2-7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "infinity-instruct-3m-0625-yi-1-5-9b",
    "bench": "alpacaeval2-lc",
    "score": 0.2054,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0625-Yi-1.5-9B' (mode 'community'), length_controlled_winrate = 20.54 (LC standard error 0.64); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 16.20; average length 1449. Config models_configs/Infinity-Instruct-3M-0625-Yi-1.5-9B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0625-Yi-1.5-9B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0625-yi-1-5-9b (HF BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0625-yi-1-5-9b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7066,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-18",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 70.66 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a42c86c61b. Identity: catalogue infinity-instruct-3m-0625-yi-1-5-9b (HF BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "infinity-instruct-3m-0613-llama3-70b",
    "bench": "alpacaeval2-lc",
    "score": 0.3153,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0613-Llama3-70B' (mode 'community'), length_controlled_winrate = 31.53 (LC standard error 0.77); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 19.27; average length 1192. Config models_configs/Infinity-Instruct-3M-0613-Llama3-70B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0613-Llama3-70B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0613-llama3-70b (HF BAAI/Infinity-Instruct-3M-0613-Llama3-70B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0613-mistral-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.255,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Infinity-Instruct-3M-0613-Mistral-7B' (mode 'community'), length_controlled_winrate = 25.50 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.75; average length 1180. Config models_configs/Infinity-Instruct-3M-0613-Mistral-7B/configs.yaml: model_name 'baai/Infinity-Instruct-3M-0613-Mistral-7B', fn_completions 'openai_completions'. Identity: catalogue infinity-instruct-3m-0613-mistral-7b (HF BAAI/Infinity-Instruct-3M-0613-Mistral-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "infinity-instruct-3m-0613-mistral-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.701,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-06-22",
    "note": "Open PT-LLM Leaderboard row 'BAAI/Infinity-Instruct-3M-0613-Mistral-7B' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 70.1 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha c1bec5cb82. Identity: catalogue infinity-instruct-3m-0613-mistral-7b (HF BAAI/Infinity-Instruct-3M-0613-Mistral-7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-8b-instruct-skillmix",
    "bench": "alpacaeval2-lc",
    "score": 0.4276,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Llama-3-8B-Instruct-SkillMix' (mode 'community'), length_controlled_winrate = 42.76 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 44.63; average length 3141. Config models_configs/Llama-3-8B-Instruct-SkillMix/configs.yaml: model_name 'PrincetonPLI/Llama-3-8B-Instruct-SkillMix', fn_completions 'vllm_local_completions'. Identity: catalogue llama-3-8b-instruct-skillmix (HF PrincetonPLI/Llama-3-8B-Instruct-SkillMix).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpo-llama-3-8b-instruct-gpm-2b",
    "bench": "alpacaeval2-lc",
    "score": 0.3843,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'GPO-Llama-3-8B-Instruct-GPM-2B' (mode 'community'), length_controlled_winrate = 38.43 (LC standard error 0.80); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 48.87; average length 2613. Config models_configs/GPO-Llama-3-8B-Instruct-GPM-2B/configs.yaml: model_name 'general-preference/GPO-Llama-3-8B-Instruct-GPM-2B', fn_completions 'vllm_local_completions'. Identity: catalogue gpo-llama-3-8b-instruct-gpm-2b (HF general-preference/GPO-Llama-3-8B-Instruct-GPM-2B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "sppo-llama-3-8b-instruct-gpm-2b",
    "bench": "alpacaeval2-lc",
    "score": 0.353,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'SPPO-Llama-3-8B-Instruct-GPM-2B' (mode 'community'), length_controlled_winrate = 35.30 (LC standard error 0.81); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 45.44; average length 2490. Config models_configs/SPPO-Llama-3-8B-Instruct-GPM-2B/configs.yaml: model_name 'general-preference/SPPO-Llama-3-8B-Instruct-GPM-2B', fn_completions 'vllm_local_completions'. Identity: catalogue sppo-llama-3-8b-instruct-gpm-2b (HF general-preference/SPPO-Llama-3-8B-Instruct-GPM-2B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "fsfairx-zephyr-chat-v0-1",
    "bench": "alpacaeval2-lc",
    "score": 0.3479,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'FsfairX-Zephyr-Chat-v0.1' (mode 'community'), length_controlled_winrate = 34.79 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 35.95; average length 2275. Config models_configs/FsfairX-Zephyr-Chat-v0.1/configs.yaml: model_name 'sfairXC/FsfairX-Zephyr-Chat-v0.1', fn_completions 'huggingface_local_completions'. Identity: catalogue fsfairx-zephyr-chat-v0-1 (HF sfairXC/FsfairX-Zephyr-Chat-v0.1).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral7b-pairrm-sppo-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.3182,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'SPPO-Mistral7B-PairRM-ExPO' (mode 'community'), length_controlled_winrate = 31.82 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 35.44; average length 2288. Config models_configs/SPPO-Mistral7B-PairRM-ExPO/configs.yaml: model_name 'chujiezheng/Mistral7B-PairRM-SPPO-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue mistral7b-pairrm-sppo-expo (HF chujiezheng/Mistral7B-PairRM-SPPO-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral7b-pairrm-sppo-expo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6298,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-09-21",
    "note": "Open PT-LLM Leaderboard row 'chujiezheng/Mistral7B-PairRM-SPPO-ExPO' (source=hub, precision=bfloat16, type=fine-tuned/fp on domain-specific datasets); published average 62.98 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 90d1dc0e7f. Identity: catalogue mistral7b-pairrm-sppo-expo (HF chujiezheng/Mistral7B-PairRM-SPPO-ExPO).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "internlm2-chat-20b-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.2723,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'internlm2-chat-20b-ExPO' (mode 'community'), length_controlled_winrate = 27.23 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 46.19; average length 3335. Config models_configs/internlm2-chat-20b-ExPO/configs.yaml: model_name 'chujiezheng/internlm2-chat-20b-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue internlm2-chat-20b-expo (HF chujiezheng/internlm2-chat-20b-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "internlm2-chat-7b-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.2267,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'internlm2-chat-7b-ExPO' (mode 'community'), length_controlled_winrate = 22.67 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 28.07; average length 2390. Config models_configs/internlm2-chat-7b-ExPO/configs.yaml: model_name 'chujiezheng/internlm2-chat-7b-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue internlm2-chat-7b-expo (HF chujiezheng/internlm2-chat-7b-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-70b-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.2572,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-70b-ExPO' (mode 'community'), length_controlled_winrate = 25.72 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 22.98; average length 1738. Config models_configs/tulu-2-dpo-70b-ExPO/configs.yaml: model_name 'chujiezheng/tulu-2-dpo-70b-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue tulu-2-dpo-70b-expo (HF chujiezheng/tulu-2-dpo-70b-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-13b-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.1759,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-13b-ExPO' (mode 'community'), length_controlled_winrate = 17.59 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 15.55; average length 1649. Config models_configs/tulu-2-dpo-13b-ExPO/configs.yaml: model_name 'chujiezheng/tulu-2-dpo-13b-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue tulu-2-dpo-13b-expo (HF chujiezheng/tulu-2-dpo-13b-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "tulu-2-dpo-7b-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.1168,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'tulu-2-dpo-7b-ExPO' (mode 'community'), length_controlled_winrate = 11.68 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 11.53; average length 1742. Config models_configs/tulu-2-dpo-7b-ExPO/configs.yaml: model_name 'chujiezheng/tulu-2-dpo-7b-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue tulu-2-dpo-7b-expo (HF chujiezheng/tulu-2-dpo-7b-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "starling-lm-7b-alpha-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.1947,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Starling-LM-7B-alpha-ExPO' (mode 'community'), length_controlled_winrate = 19.47 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 18.18; average length 1821. Config models_configs/Starling-LM-7B-alpha-ExPO/configs.yaml: model_name 'chujiezheng/Starling-LM-7B-alpha-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue starling-lm-7b-alpha-expo (HF chujiezheng/Starling-LM-7B-alpha-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-alpha-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.1357,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'zephyr-7b-alpha-ExPO' (mode 'community'), length_controlled_winrate = 13.57 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 10.56; average length 1248. Config models_configs/zephyr-7b-alpha-ExPO/configs.yaml: model_name 'chujiezheng/zephyr-7b-alpha-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue zephyr-7b-alpha-expo (HF chujiezheng/zephyr-7b-alpha-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "zephyr-7b-beta-expo",
    "bench": "alpacaeval2-lc",
    "score": 0.14,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'zephyr-7b-beta-ExPO' (mode 'community'), length_controlled_winrate = 14.00 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 11.06; average length 1405. Config models_configs/zephyr-7b-beta-ExPO/configs.yaml: model_name 'chujiezheng/zephyr-7b-beta-ExPO', fn_completions 'vllm_local_completions'. Identity: catalogue zephyr-7b-beta-expo (HF chujiezheng/zephyr-7b-beta-ExPO).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "contextual-kto-mistral-pairrm",
    "bench": "alpacaeval2-lc",
    "score": 0.2971,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Contextual-KTO-Mistral-PairRM' (mode 'verified'), length_controlled_winrate = 29.71 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 33.23; average length 2521. Config models_configs/Contextual-KTO-Mistral-PairRM/configs.yaml: model_name 'ContextualAI/Contextual_KTO_Mistral_PairRM', fn_completions 'vllm_local_completions'. Identity: catalogue contextual-kto-mistral-pairrm (HF ContextualAI/Contextual_KTO_Mistral_PairRM).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "lmcocktail-10-7b-v1",
    "bench": "alpacaeval1-winrate",
    "score": 0.9222,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'LMCocktail-10.7B-v1' (mode 'community'), win_rate = 92.22 (standard error 0.94); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1203. Config models_configs/LMCocktail-10.7B-v1/configs.yaml: model_name 'Yhyu13/LMCocktail-10.7B-v1', fn_completions 'vllm_local_completions'. Identity: catalogue lmcocktail-10-7b-v1 (HF Yhyu13/LMCocktail-10.7B-v1).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "lmcocktail-10-7b-v1",
    "bench": "alpacaeval2-lc",
    "score": 0.1895,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'LMCocktail-10.7B-v1' (mode 'community'), length_controlled_winrate = 18.95 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 13.15; average length 1203. Config models_configs/LMCocktail-10.7B-v1/configs.yaml: model_name 'Yhyu13/LMCocktail-10.7B-v1', fn_completions 'vllm_local_completions'. Identity: catalogue lmcocktail-10-7b-v1 (HF Yhyu13/LMCocktail-10.7B-v1).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "deita-7b-v1-0",
    "bench": "alpacaeval1-winrate",
    "score": 0.9006,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'deita-7b-v1.0' (mode 'community'), win_rate = 90.06 (standard error 1.06); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1417. Config models_configs/deita-7b-v1.0/configs.yaml: model_name 'hkust-nlp/deita-7b-v1.0', fn_completions 'vllm_local_completions'. Identity: catalogue deita-7b-v1-0 (HF hkust-nlp/deita-7b-v1.0).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "deita-7b-v1-0",
    "bench": "alpacaeval2-lc",
    "score": 0.1606,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'deita-7b-v1.0' (mode 'community'), length_controlled_winrate = 16.06 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 12.65; average length 1417. Config models_configs/deita-7b-v1.0/configs.yaml: model_name 'hkust-nlp/deita-7b-v1.0', fn_completions 'vllm_local_completions'. Identity: catalogue deita-7b-v1-0 (HF hkust-nlp/deita-7b-v1.0).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "deita-7b-v1-0",
    "bench": "open-pt-llm-avg9",
    "score": 0.6632,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-25",
    "note": "Open PT-LLM Leaderboard row 'hkust-nlp/deita-7b-v1.0' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 66.32 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha de16c0994d. Identity: catalogue deita-7b-v1-0 (HF hkust-nlp/deita-7b-v1.0).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "causallm-14b",
    "bench": "alpacaeval1-winrate",
    "score": 0.8826,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'causallm-14b' (mode 'community'), win_rate = 88.26 (standard error 1.12); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1391. Config models_configs/causallm-14b/configs.yaml: model_name 'CausalLM/14B', fn_completions 'vllm_local_completions'. Identity: catalogue causallm-14b (HF CausalLM/14B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "causallm-14b",
    "bench": "alpacaeval2-lc",
    "score": 0.1572,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'causallm-14b' (mode 'community'), length_controlled_winrate = 15.72 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 11.15; average length 1391. Config models_configs/causallm-14b/configs.yaml: model_name 'CausalLM/14B', fn_completions 'vllm_local_completions'. Identity: catalogue causallm-14b (HF CausalLM/14B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "mistral-orpo-beta",
    "bench": "alpacaeval2-lc",
    "score": 0.1472,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'mistral-orpo-beta' (mode 'community'), length_controlled_winrate = 14.72 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 12.57; average length 1636. Config models_configs/mistral-orpo-beta/configs.yaml: model_name 'kaist-ai/mistral-orpo-beta', fn_completions 'huggingface_local_completions'. Identity: catalogue mistral-orpo-beta (HF kaist-ai/mistral-orpo-beta).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minotaur-13b-fixed",
    "bench": "alpacaeval1-winrate",
    "score": 0.6602,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'minotaur-13b' (mode 'community'), win_rate = 66.02 (standard error 1.66); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 881. Config models_configs/minotaur-13b/configs.yaml: model_name 'openaccess-ai-collective/minotaur-13b-fixed', fn_completions 'huggingface_local_completions'. Identity: catalogue minotaur-13b-fixed (HF openaccess-ai-collective/minotaur-13b-fixed).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minotaur-13b-fixed",
    "bench": "alpacaeval2-lc",
    "score": 0.1147,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'minotaur-13b' (mode 'community'), length_controlled_winrate = 11.47 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 5.74; average length 881. Config models_configs/minotaur-13b/configs.yaml: model_name 'openaccess-ai-collective/minotaur-13b-fixed', fn_completions 'huggingface_local_completions'. Identity: catalogue minotaur-13b-fixed (HF openaccess-ai-collective/minotaur-13b-fixed).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-13b-v1-3",
    "bench": "alpacaeval1-winrate",
    "score": 0.8211,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'vicuna-13b-v1.3' (mode 'verified'), win_rate = 82.11 (standard error 1.35); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1132. Config models_configs/vicuna-13b-v1.3/configs.yaml: model_name 'lmsys/vicuna-13b-v1.3', fn_completions 'huggingface_local_completions'. Identity: catalogue vicuna-13b-v1-3 (HF lmsys/vicuna-13b-v1.3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-13b-v1-3",
    "bench": "alpacaeval2-lc",
    "score": 0.1084,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'vicuna-13b-v1.3' (mode 'verified'), length_controlled_winrate = 10.84 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 7.14; average length 1132. Config models_configs/vicuna-13b-v1.3/configs.yaml: model_name 'lmsys/vicuna-13b-v1.3', fn_completions 'huggingface_local_completions'. Identity: catalogue vicuna-13b-v1-3 (HF lmsys/vicuna-13b-v1.3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-7b-v1-3",
    "bench": "alpacaeval1-winrate",
    "score": 0.7684,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'vicuna-7b-v1.3' (mode 'verified'), win_rate = 76.84 (standard error 1.49); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1110. Config models_configs/vicuna-7b-v1.3/configs.yaml: model_name 'lmsys/vicuna-7b-v1.3', fn_completions 'huggingface_local_completions'. Identity: catalogue vicuna-7b-v1-3 (HF lmsys/vicuna-7b-v1.3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "vicuna-7b-v1-3",
    "bench": "alpacaeval2-lc",
    "score": 0.0716,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'vicuna-7b-v1.3' (mode 'verified'), length_controlled_winrate = 7.16 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 4.64; average length 1110. Config models_configs/vicuna-7b-v1.3/configs.yaml: model_name 'lmsys/vicuna-7b-v1.3', fn_completions 'huggingface_local_completions'. Identity: catalogue vicuna-7b-v1-3 (HF lmsys/vicuna-7b-v1.3).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "platolm-7b",
    "bench": "alpacaeval1-winrate",
    "score": 0.8194,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'platolm-7b' (mode 'community'), win_rate = 81.94 (standard error 1.35); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1344. Config models_configs/platolm-7b/configs.yaml: model_name 'FreedomIntelligence/PlatoLM-7b', fn_completions 'huggingface_local_completions'. Identity: catalogue platolm-7b (HF FreedomIntelligence/PlatoLM-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "platolm-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.1054,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'platolm-7b' (mode 'community'), length_controlled_winrate = 10.54 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 6.32; average length 1344. Config models_configs/platolm-7b/configs.yaml: model_name 'FreedomIntelligence/PlatoLM-7b', fn_completions 'huggingface_local_completions'. Identity: catalogue platolm-7b (HF FreedomIntelligence/PlatoLM-7B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "nous-hermes-13b",
    "bench": "alpacaeval1-winrate",
    "score": 0.6547,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'nous-hermes-13b' (mode 'verified'), win_rate = 65.47 (standard error 1.67); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 844. Config models_configs/nous-hermes-13b/configs.yaml: model_name 'NousResearch/Nous-Hermes-13b', fn_completions 'huggingface_local_completions'. Identity: catalogue nous-hermes-13b (HF NousResearch/Nous-Hermes-13b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "nous-hermes-13b",
    "bench": "alpacaeval2-lc",
    "score": 0.0972,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'nous-hermes-13b' (mode 'verified'), length_controlled_winrate = 9.72 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 5.41; average length 844. Config models_configs/nous-hermes-13b/configs.yaml: model_name 'NousResearch/Nous-Hermes-13b', fn_completions 'huggingface_local_completions'. Identity: catalogue nous-hermes-13b (HF NousResearch/Nous-Hermes-13b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "nous-hermes-13b",
    "bench": "open-pt-llm-avg9",
    "score": 0.557,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Nous-Hermes-13b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 55.7 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 24e8c03148. Identity: catalogue nous-hermes-13b (HF NousResearch/Nous-Hermes-13b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "minichat-1-5-3b",
    "bench": "alpacaeval1-winrate",
    "score": 0.7855,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'minichat-1.5-3b' (mode 'community'), win_rate = 78.55 (standard error 1.45); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1545. Config models_configs/minichat-1.5-3b/configs.yaml: model_name 'GeneZC/MiniChat-1.5-3B', fn_completions 'huggingface_local_completions'. Identity: catalogue minichat-1-5-3b (HF GeneZC/MiniChat-1.5-3B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minichat-1-5-3b",
    "bench": "alpacaeval2-lc",
    "score": 0.077,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'minichat-1.5-3b' (mode 'community'), length_controlled_winrate = 7.70 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 6.55; average length 1545. Config models_configs/minichat-1.5-3b/configs.yaml: model_name 'GeneZC/MiniChat-1.5-3B', fn_completions 'huggingface_local_completions'. Identity: catalogue minichat-1-5-3b (HF GeneZC/MiniChat-1.5-3B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minichat-1-5-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Open PT-LLM Leaderboard row 'GeneZC/MiniChat-1.5-3B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 0.0 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 91b66d3602. Identity: catalogue minichat-1-5-3b (HF GeneZC/MiniChat-1.5-3B).",
    "invalid": "All nine published task scores are 0.0 in this Open PT row (failed run with no parsed answers), not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "minichat-3b",
    "bench": "alpacaeval1-winrate",
    "score": 0.4882,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'minichat-3b' (mode 'community'), win_rate = 48.82 (standard error 1.76); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 868. Config models_configs/minichat-3b/configs.yaml: model_name 'GeneZC/MiniChat-3B', fn_completions 'huggingface_local_completions'. Identity: catalogue minichat-3b (HF GeneZC/MiniChat-3B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minichat-3b",
    "bench": "alpacaeval2-lc",
    "score": 0.0573,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'minichat-3b' (mode 'community'), length_controlled_winrate = 5.73 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 3.01; average length 868. Config models_configs/minichat-3b/configs.yaml: model_name 'GeneZC/MiniChat-3B', fn_completions 'huggingface_local_completions'. Identity: catalogue minichat-3b (HF GeneZC/MiniChat-3B).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "minichat-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Open PT-LLM Leaderboard row 'GeneZC/MiniChat-3B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 0.0 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 5b9109277d. Identity: catalogue minichat-3b (HF GeneZC/MiniChat-3B).",
    "invalid": "All nine published task scores are 0.0 in this Open PT row (failed run with no parsed answers), not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "baize-v2-13b",
    "bench": "alpacaeval1-winrate",
    "score": 0.6696,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'baize-v2-13b' (mode 'community'), win_rate = 66.96 (standard error 1.66); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 930. Config models_configs/baize-v2-13b/configs.yaml: model_name 'project-baize/baize-v2-13b', fn_completions 'huggingface_local_completions'. Identity: catalogue baize-v2-13b (HF project-baize/baize-v2-13b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "baize-v2-13b",
    "bench": "alpacaeval2-lc",
    "score": 0.0701,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'baize-v2-13b' (mode 'community'), length_controlled_winrate = 7.01 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 4.59; average length 930. Config models_configs/baize-v2-13b/configs.yaml: model_name 'project-baize/baize-v2-13b', fn_completions 'huggingface_local_completions'. Identity: catalogue baize-v2-13b (HF project-baize/baize-v2-13b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "baize-v2-7b",
    "bench": "alpacaeval1-winrate",
    "score": 0.6385,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'baize-v2-7b' (mode 'community'), win_rate = 63.85 (standard error 1.69); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1127. Config models_configs/baize-v2-7b/configs.yaml: model_name 'project-baize/baize-v2-7b', fn_completions 'huggingface_local_completions'. Identity: catalogue baize-v2-7b (HF project-baize/baize-v2-7b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "baize-v2-7b",
    "bench": "alpacaeval2-lc",
    "score": 0.0438,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'baize-v2-7b' (mode 'community'), length_controlled_winrate = 4.38 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 3.40; average length 1127. Config models_configs/baize-v2-7b/configs.yaml: model_name 'project-baize/baize-v2-7b', fn_completions 'huggingface_local_completions'. Identity: catalogue baize-v2-7b (HF project-baize/baize-v2-7b).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "ghost-7b-alpha",
    "bench": "alpacaeval1-winrate",
    "score": 0.7044,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'ghost-7b-alpha' (mode 'community'), win_rate = 70.44 (standard error 1.62); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1681. Config models_configs/ghost-7b-alpha/configs.yaml: model_name 'ghost-x/ghost-7b-alpha', fn_completions 'huggingface_local_completions'. Identity: catalogue ghost-7b-alpha (HF ghost-x/ghost-7b-alpha).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "ghost-7b-alpha",
    "bench": "alpacaeval2-lc",
    "score": 0.0685,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'ghost-7b-alpha' (mode 'community'), length_controlled_winrate = 6.85 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 6.11; average length 1681. Config models_configs/ghost-7b-alpha/configs.yaml: model_name 'ghost-x/ghost-7b-alpha', fn_completions 'huggingface_local_completions'. Identity: catalogue ghost-7b-alpha (HF ghost-x/ghost-7b-alpha).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "oasst-pythia-12b-sft-v8-7k-steps",
    "bench": "alpacaeval1-winrate",
    "score": 0.4186,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'pythia-12b-mix-sft' (mode 'verified'), win_rate = 41.86 (standard error 1.74); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 913. Config models_configs/pythia-12b-mix-sft/configs.yaml: model_name 'OpenAssistant/pythia-12b-sft-v8-7k-steps', fn_completions 'huggingface_local_completions'. Identity: catalogue oasst-pythia-12b-sft-v8-7k-steps (HF OpenAssistant/pythia-12b-sft-v8-7k-steps).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "oasst-pythia-12b-sft-v8-7k-steps",
    "bench": "alpacaeval2-lc",
    "score": 0.0422,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'pythia-12b-mix-sft' (mode 'verified'), length_controlled_winrate = 4.22 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 2.58; average length 913. Config models_configs/pythia-12b-mix-sft/configs.yaml: model_name 'OpenAssistant/pythia-12b-sft-v8-7k-steps', fn_completions 'huggingface_local_completions'. Identity: catalogue oasst-pythia-12b-sft-v8-7k-steps (HF OpenAssistant/pythia-12b-sft-v8-7k-steps).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "baichuan-13b-chat",
    "bench": "alpacaeval1-winrate",
    "score": 0.218,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'baichuan-13b-chat' (mode 'community'), win_rate = 21.80 (standard error 1.45); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1727. Config models_configs/baichuan-13b-chat/configs.yaml: model_name 'baichuan-inc/Baichuan-13B-Chat', fn_completions 'huggingface_local_completions'. Identity: catalogue baichuan-13b-chat (HF baichuan-inc/Baichuan-13B-Chat).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "baichuan-13b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.0206,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'baichuan-13b-chat' (mode 'community'), length_controlled_winrate = 2.06 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 1.99; average length 1727. Config models_configs/baichuan-13b-chat/configs.yaml: model_name 'baichuan-inc/Baichuan-13B-Chat', fn_completions 'huggingface_local_completions'. Identity: catalogue baichuan-13b-chat (HF baichuan-inc/Baichuan-13B-Chat).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen-7b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.5814,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen-7B [bfloat16]' (source=hub, precision=bfloat16, type=pretrained); published average 58.14 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha ef3c5c9c57. Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen-7b-base",
    "bench": "arc-challenge",
    "score": 0.49146757679180886,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "arc-easy",
    "score": 0.6519360269360269,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "boolq",
    "score": 0.745565749235474,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "hellaswag",
    "score": 0.8884684325831508,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "lambada-openai",
    "score": 0.6966815447312246,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "openbookqa",
    "score": 0.322,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "piqa",
    "score": 0.7399347116430903,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "sciq",
    "score": 0.932,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "social-iqa",
    "score": 0.49437052200614123,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-base",
    "bench": "winogrande",
    "score": 0.65982636148382,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=Qwen/Qwen-7B,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-base (HF Qwen/Qwen-7B).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.5987,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-24",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen-7B-Chat [bfloat16]' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 59.87 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 8d24619bab. Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "arc-challenge",
    "score": 0.4667235494880546,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "arc-easy",
    "score": 0.6447811447811448,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "boolq",
    "score": 0.7168195718654434,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "hellaswag",
    "score": 0.8497311292571201,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "lambada-openai",
    "score": 0.6547642150203765,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "openbookqa",
    "score": 0.356,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "piqa",
    "score": 0.7872687704026116,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "sciq",
    "score": 0.907,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "social-iqa",
    "score": 0.4703172978505629,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "qwen-7b-chat",
    "bench": "winogrande",
    "score": 0.6858721389108129,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/Qwen-Qwen-7B-Chat.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=Qwen/Qwen-7B-Chat,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue qwen-7b-chat (HF Qwen/Qwen-7B-Chat).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.4682,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'baichuan-inc/Baichuan2-7B-Base' (source=hub, precision=bfloat16, type=pretrained); published average 46.82 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f9d4d8dd2f. Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "arc-challenge",
    "score": 0.431740614334471,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "arc-easy",
    "score": 0.7281144781144782,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "boolq",
    "score": 0.7308868501529052,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "hellaswag",
    "score": 0.7228639713204541,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "lambada-openai",
    "score": 0.7098777411216767,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "openbookqa",
    "score": 0.304,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "piqa",
    "score": 0.7616974972796517,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "sciq",
    "score": 0.946,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "social-iqa",
    "score": 0.41760491299897645,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "baichuan2-7b-base",
    "bench": "winogrande",
    "score": 0.675611681136543,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/baichuan-inc_Baichuan2-7B-Base.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=baichuan-inc/Baichuan2-7B-Base,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=True. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue baichuan2-7b-base (HF baichuan-inc/Baichuan2-7B-Base).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "open-pt-llm-avg9",
    "score": 0.2474,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-2.8b-deduped' (source=hub, precision=float16, type=pretrained); published average 24.74 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 7d977fed8c. Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "arc-challenge",
    "score": 0.3293515358361775,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "arc-easy",
    "score": 0.5909090909090909,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "boolq",
    "score": 0.6412844036697247,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "hellaswag",
    "score": 0.5944035052778331,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "lambada-openai",
    "score": 0.6514651659227635,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "openbookqa",
    "score": 0.238,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "piqa",
    "score": 0.7410228509249184,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "sciq",
    "score": 0.882,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "social-iqa",
    "score": 0.4094165813715456,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "winogrande",
    "score": 0.5824782951854776,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "truthfulqa-mc2",
    "score": 0.3555711185495532,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/EleutherAI-pythia-2.8b-deduped.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=EleutherAI/pythia-2.8b-deduped,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue pythia-2-8b-deduped (HF EleutherAI/pythia-2.8b-deduped).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "open-pt-llm-avg9",
    "score": 0.3064,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'openlm-research/open_llama_3b_v2' (source=hub, precision=float16, type=pretrained); published average 30.64 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha bce5d60d3b. Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "arc-challenge",
    "score": 0.3609215017064846,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "arc-easy",
    "score": 0.63510101010101,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "boolq",
    "score": 0.6568807339449542,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "hellaswag",
    "score": 0.6998605855407289,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "lambada-openai",
    "score": 0.6673782262759558,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "openbookqa",
    "score": 0.26,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "piqa",
    "score": 0.7665941240478781,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "sciq",
    "score": 0.924,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "social-iqa",
    "score": 0.4119754350051177,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "winogrande",
    "score": 0.6290449881610103,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-3b-v2",
    "bench": "truthfulqa-mc2",
    "score": 0.3458747299959986,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_3b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=False,pretrained=openlm-research/open_llama_3b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-3b-v2 (HF openlm-research/open_llama_3b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "open-pt-llm-avg9",
    "score": 0.392,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'openlm-research/open_llama_7b_v2' (source=hub, precision=float16, type=pretrained); published average 39.2 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha e5961def23. Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "arc-challenge",
    "score": 0.42406143344709896,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "arc-easy",
    "score": 0.6965488215488216,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "boolq",
    "score": 0.7140672782874617,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "hellaswag",
    "score": 0.7464648476399124,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "lambada-openai",
    "score": 0.7104599262565496,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "openbookqa",
    "score": 0.302,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "piqa",
    "score": 0.7916213275299239,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "sciq",
    "score": 0.938,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "social-iqa",
    "score": 0.4196519959058342,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "winogrande",
    "score": 0.6582478295185478,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "open-llama-7b-v2",
    "bench": "truthfulqa-mc2",
    "score": 0.3456877328963021,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/openlm-research-open_llama_7b_v2.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=False,pretrained=openlm-research/open_llama_7b_v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=bfloat16. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue open-llama-7b-v2 (HF openlm-research/open_llama_7b_v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "open-pt-llm-avg9",
    "score": 0.306,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-base-alpha-3b-v2' (source=hub, precision=float16, type=pretrained); published average 30.6 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 997d95943e. Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "arc-challenge",
    "score": 0.3506825938566553,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "arc-easy",
    "score": 0.6325757575757576,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "boolq",
    "score": 0.645565749235474,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "hellaswag",
    "score": 0.6858195578570006,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "lambada-openai",
    "score": 0.7025033960799534,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "openbookqa",
    "score": 0.264,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "piqa",
    "score": 0.7600652883569097,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "sciq",
    "score": 0.921,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "social-iqa",
    "score": 0.4247697031729785,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "winogrande",
    "score": 0.6211523283346487,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-3b-v2",
    "bench": "truthfulqa-mc2",
    "score": 0.35868737415331753,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-3b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-3b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-3b-v2 (HF stabilityai/stablelm-base-alpha-3b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "open-pt-llm-avg9",
    "score": 0.4458,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-base-alpha-7b-v2' (source=hub, precision=float16, type=pretrained); published average 44.58 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 6d176503c1. Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "arc-challenge",
    "score": 0.4052901023890785,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "arc-easy",
    "score": 0.6910774410774411,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "boolq",
    "score": 0.7030581039755351,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "hellaswag",
    "score": 0.7426807408882693,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "lambada-openai",
    "score": 0.7418979235396856,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "openbookqa",
    "score": 0.304,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "piqa",
    "score": 0.7845484221980413,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "sciq",
    "score": 0.939,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "social-iqa",
    "score": 0.4242579324462641,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "winogrande",
    "score": 0.6882399368587214,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "stablelm-base-alpha-7b-v2",
    "bench": "truthfulqa-mc2",
    "score": 0.36457253410937596,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/stablelm-base-alpha-7b-v2.json",
    "by": "lab",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=stabilityai/stablelm-base-alpha-7b-v2,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue stablelm-base-alpha-7b-v2 (HF stabilityai/stablelm-base-alpha-7b-v2).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "gemini-1-5-pro-exp-0801",
    "bench": "cruxeval-zeroeval",
    "score": 0.7488,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0801'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.12. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0801 (API id gemini-1.5-pro-exp-0801).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0801",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8553,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0801'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.22. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0801 (API id gemini-1.5-pro-exp-0801).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0801",
    "bench": "zebralogic-zeroeval",
    "score": 0.252,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0801'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0801 (API id gemini-1.5-pro-exp-0801).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0801",
    "bench": "gsm8k-zeroeval",
    "score": 0.95,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0801'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0801 (API id gemini-1.5-pro-exp-0801).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0801",
    "bench": "livebench-20240726",
    "score": 0.5222,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-pro-exp-0801'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 52.22. Category means: math 47.5, coding 41.2, reasoning 48.7, language 47.0, instruction_following 78.8, data_analysis 50.2. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue gemini-1-5-pro-exp-0801 (API id gemini-1.5-pro-exp-0801).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "math-l5-zeroeval",
    "score": 0.681,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0827'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=1.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "cruxeval-zeroeval",
    "score": 0.7962,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0827'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.25. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8614,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0827'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.26. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "zebralogic-zeroeval",
    "score": 0.305,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-pro-exp-0827'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=0.80. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "livebench-20240726",
    "score": 0.5378,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-pro-exp-0827'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 53.78. Category means: math 56.3, coding 40.9, reasoning 49.3, language 49.3, instruction_following 75.9, data_analysis 50.8. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemini-1-5-pro-exp-0827",
    "bench": "aider-code-editing",
    "score": 0.669,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-08-28",
    "note": "Published entry 'gemini-1.5-pro-exp-0827'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 66.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff-fenced; Aider version=0.53.1-dev; evaluator commit=d8adc75; command=aider --model gemini/gemini-1.5-pro-exp-0827; run=2024-08-28-07-10-50--gemini-1.5-pro-exp-0827-diff-fenced. Identity: catalogue gemini-1-5-pro-exp-0827 (API id gemini-1.5-pro-exp-0827). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "math-l5-zeroeval",
    "score": 0.5451,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-exp-0827'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=6.80. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "cruxeval-zeroeval",
    "score": 0.745,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-exp-0827'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.38. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8211,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-exp-0827'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=1.48. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "zebralogic-zeroeval",
    "score": 0.25,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'gemini-1.5-flash-exp-0827'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=8.50. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "livebench-20240726",
    "score": 0.4687,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-flash-exp-0827'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 46.87. Category means: math 36.3, coding 40.6, reasoning 47.3, language 31.0, instruction_following 78.1, data_analysis 47.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemini-1-5-flash-exp-0827",
    "bench": "aider-code-editing",
    "score": 0.526,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-08-27",
    "note": "Published entry 'gemini-1.5-flash-exp-0827'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 52.6 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.53.1-dev; evaluator commit=d8adc75; command=aider --model gemini/gemini-1.5-flash-exp-0827; run=2024-08-27-19-20-19--gemini-1.5-flash-exp-0827. Identity: catalogue gemini-1-5-flash-exp-0827 (API id gemini-1.5-flash-exp-0827). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-1-5-flash-8b-exp-0827",
    "bench": "livebench-20240726",
    "score": 0.3702,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'gemini-1.5-flash-8b-exp-0827'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 37.02. Category means: math 33.2, coding 28.7, reasoning 33.3, language 22.5, instruction_following 69.0, data_analysis 35.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue gemini-1-5-flash-8b-exp-0827 (API id gemini-1.5-flash-8b-exp-0827).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gemini-1-5-flash-8b-exp-0827",
    "bench": "aider-code-editing",
    "score": 0.383,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-08-27",
    "note": "Published entry 'gemini-1.5-flash-8b-exp-0827'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 38.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.53.1-dev; evaluator commit=d8adc75; command=aider --model gemini/gemini-1.5-flash-8b-exp-0827; run=2024-08-27-19-42-05--gemini-1.5-flash-8b-exp-0827. Identity: catalogue gemini-1-5-flash-8b-exp-0827 (API id gemini-1.5-flash-8b-exp-0827). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-1-5-flash-8b-exp-0924",
    "bench": "aider-code-editing",
    "score": 0.383,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-09-24",
    "note": "Published entry 'gemini-1.5-flash-8b-exp-0924'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 38.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.57.2.dev; evaluator commit=86faaa6; command=aider --model gemini/gemini-1.5-flash-8b-exp-0924; run=2024-09-24-15-18-59--gemini-1.5-flash-8b-exp-0924-whole. Identity: catalogue gemini-1-5-flash-8b-exp-0924 (API id gemini-1.5-flash-8b-exp-0924). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-exp-1114",
    "bench": "aider-code-editing",
    "score": 0.609,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-15",
    "note": "Published entry 'gemini-exp-1114'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 60.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.63.2.dev; evaluator commit=0bf17a4; command=aider --model gemini/gemini-exp-1114; run=2024-11-15-20-33-31--gemini-exp-1114-diff. Identity: catalogue gemini-exp-1114 (API id gemini-exp-1114). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-exp-1121",
    "bench": "aider-code-editing",
    "score": 0.579,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-21",
    "note": "Published entry 'gemini-exp-1121'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 57.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.63.3.dev; evaluator commit=e94961a; command=aider --model gemini/gemini-exp-1121; run=2024-11-21-17-46-36--gemini-exp-1121-diff. Identity: catalogue gemini-exp-1121 (API id gemini-exp-1121). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-exp-1206",
    "bench": "aider-polyglot",
    "score": 0.382,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-22",
    "note": "Published entry 'gemini-exp-1206'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=b1bc2f8; command=aider --model gemini/gemini-exp-1206; run=2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Identity: catalogue gemini-exp-1206 (API id gemini-exp-1206). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-exp-1206",
    "bench": "aider-code-editing",
    "score": 0.805,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-08",
    "note": "Published entry 'gemini-exp-1206 (whole)'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 80.5 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.67.1.dev; evaluator commit=f2d2ab5; command=aider --model gemini/gemini-exp-1206; run=2024-12-08-21-39-06--gemini-exp-1206-whole. Higher of the 2 full runs (gemini-exp-1206 (whole) 80.5; gemini-exp-1206 (diff) 69.2). Identity: catalogue gemini-exp-1206 (API id gemini-exp-1206). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "aider-polyglot",
    "score": 0.222,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-22",
    "note": "Published entry 'gemini-2.0-flash-exp'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.69.2.dev; evaluator commit=b1bc2f8; command=aider --model gemini/gemini-2.0-flash-exp; run=2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Identity: catalogue gemini-2-0-flash-exp (API id gemini-2.0-flash-exp). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "aider-code-editing",
    "score": 0.699,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-11",
    "note": "Published entry 'gemini-2.0-flash-exp'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 69.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=diff; Aider version=0.68.1.dev; evaluator commit=fcb2bac-dirty, 02e7e31-dirty; command=aider --model gemini/gemini-2.0-flash-exp; run=2024-12-11-21-07-00--gemini-2-flash-diff. Identity: catalogue gemini-2-0-flash-exp (API id gemini-2.0-flash-exp). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "mistral-small-2402",
    "bench": "livebench-20240726",
    "score": 0.3219,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'mistral-small-2402'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 32.19. Category means: math 28.1, coding 21.2, reasoning 26.0, language 22.1, instruction_following 63.9, data_analysis 31.9. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README). Identity: catalogue mistral-small-2402 (API id mistral-small-2402).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "mistral-small-2402",
    "bench": "bfcl-v3",
    "score": 0.4632,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-04-13/score/data_overall.csv",
    "by": "independent",
    "date": "2025-04-13",
    "note": "Published entry 'Mistral-small-2402 (FC)'. BFCL v3 leaderboard snapshot 2025-04-13 in the BFCL-Result archive; field Overall Acc = 46.32%. Entry absent from the newer v3 snapshot(s) (2025-06-14, 2025-04-25), so the latest snapshot that lists it is used (earlier rows retained only when the exact entry is absent from the newer snapshot). Same v3 scoring as 2025-06-14 (post-2025-04-13 category set without Exec; identical column set). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://docs.mistral.ai/guides/model-selection/ (organisation 'Mistral AI', licence 'Proprietary'). Identity: catalogue mistral-small-2402 (API id mistral-small-2402). Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "mistral-small-2402",
    "bench": "bigcodebench-complete",
    "score": 0.39,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Mistral' / checkpoint 'Small', column 'Greedy Decoding · Original' = 0.39 (calibrated 0.413). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4 'Models for Evaluations': 'Mistral-Small | mistral-small-2402'.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "mistral-small-2402",
    "bench": "bigcodebench-instruct",
    "score": 0.318,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Mistral' / checkpoint 'Small', column 'Greedy Decoding · Original' = 0.318 (calibrated 0.321). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: Table 4 'Models for Evaluations': 'Mistral-Small | mistral-small-2402'.",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "codestral-2501",
    "bench": "aider-polyglot",
    "score": 0.111,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-01-13",
    "note": "Published entry 'Codestral 25.01'. Aider polyglot; 225 exercises; pass_rate_2 after at most two attempts (not pass@1). edit_format=whole; Aider version=0.71.2.dev; evaluator commit=0cba898-dirty; command=aider --model mistral/codestral-latest; run=2025-01-13-18-17-25--codestral-whole2. Reasoning label and command retained as published; unspecified decoding/token limits are not inferred. Identity: catalogue codestral-2501 (API id codestral-2501). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "reka-core-20240501",
    "bench": "math-l5-zeroeval",
    "score": 0.2191,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-core-20240501'. ZeroEval math-l5 (MATH Level 5, AI-MO/aimo-validation-math-level-5, 721 problems); zero-shot reasoning + JSON answer; Mode=greedy; 721 evaluated items; field=Acc; no-answer percentage=2.64. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-core-20240501",
    "bench": "cruxeval-zeroeval",
    "score": 0.4625,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-core-20240501'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-core-20240501",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.7642,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-core-20240501'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.76. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-core-20240501",
    "bench": "zebralogic-zeroeval",
    "score": 0.13,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-core-20240501'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=4.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-core-20240501",
    "bench": "gsm8k-zeroeval",
    "score": 0.8741,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "note": "Published entry 'reka-core-20240501'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.08. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit and weight revision are not in the summary and are not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "reka-core-20240501",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 45.9028,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'reka-core-20240501': WB_score.task_macro = 45.9028 (WB_score 41.0352). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). WB-Elo and pairwise rewards not used. Apache-2.0 (repository LICENSE). Date is the data commit date. Identity: catalogue reka-core-20240501 (API id reka-core-20240501).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "gpt-3-5-turbo-16k-0613",
    "bench": "alpacaeval1-winrate",
    "score": 0.9342,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'gpt-3.5-turbo-16k-0613' (mode 'verified'), win_rate = 93.42 (standard error 0.85); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 1328. Config models_configs/gpt-3.5-turbo-16k-0613/configs.yaml: model_name 'gpt-3.5-turbo-16k-0613', fn_completions 'openai_completions'. Identity: catalogue gpt-3-5-turbo-16k-0613 (API id gpt-3.5-turbo-16k-0613).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "gpt-3-5-turbo-16k-0613",
    "bench": "alpacaeval2-lc",
    "score": 0.2272,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'gpt-3.5-turbo-16k-0613' (mode 'verified'), length_controlled_winrate = 22.72 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 14.13; average length 1328. Config models_configs/gpt-3.5-turbo-16k-0613/configs.yaml: model_name 'gpt-3.5-turbo-16k-0613', fn_completions 'openai_completions'. Identity: catalogue gpt-3-5-turbo-16k-0613 (API id gpt-3.5-turbo-16k-0613).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "claude-instant-1-2",
    "bench": "alpacaeval2-lc",
    "score": 0.2561,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'claude-instant-1.2' (mode 'community'), length_controlled_winrate = 25.61 (LC standard error not published); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 16.13; average length 1112. Config models_configs/claude-instant-1.2/configs.yaml: model_name 'claude-instant-1.2', fn_completions 'anthropic_completions'. Identity: catalogue claude-instant-1-2 (API id claude-instant-1.2).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "codex-mini-latest",
    "bench": "webapp1k-duo-react-pass1",
    "score": 0.575,
    "source": "https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard/blob/f05bc19c75983e09e6c3198ff13e28c8c9362ff0/results_duo.csv",
    "by": "independent",
    "date": "2025-05-17",
    "note": "Entry 'codex-mini-latest' in results_duo.csv: mean per-problem Successes/Runs over 1000 problems (1000 runs), reproducing the Space's pass@1. Reasoning effort/decoding not published. Date = first appearance of the entry in the Space commit history (2025-05-17, per the 2026-10-09 refresh review); value read at pinned revision f05bc19c75983e09e6c3198ff13e28c8c9362ff0.",
    "source_id": "webapp1k-leaderboard"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "gpqa-diamond",
    "score": 0.8964646464646465,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-10-09",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id ZEKWTGJiWFscHimMkiUmec; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.019774484382483127. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "otis-mock-aime",
    "score": 0.9888888888888889,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_xhigh'; run id iiEPrSjWXHH6Zq9CK4Zmcr; effort xhigh; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.005362818328685681. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "frontiermath",
    "score": 0.7508771929824561,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id 2QAt4ptgGkLtcbVL3hdAvL; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.02566445410232946. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "simpleqa-verified",
    "score": 0.238,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-10-09",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id NwnrUMtveqHDYB2Um4ycmA; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.013473586661967093. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.4634146341463415,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id WmUXFQoq7GqiSgpTmfLaVv; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07884502378168105. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "epoch-mystery-games",
    "score": 0.3,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id CKm7NWXfSePfcUycRK87cB; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.04605661864718382. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "claude-haiku-5-5",
    "bench": "epoch-furniture",
    "score": 0.475,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'claude-haiku-5-5_max'; run id bZMMyetG9pAbKgpxghoTe8; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06446761548504554. Catalogue claude-haiku-5-5 (added 2026-10-10); Epoch model_metadata display 'Claude Haiku 5.5 (max)', release 2026-10-07.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "gpqa-diamond",
    "score": 0.8983585858585859,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2026-10-09",
    "note": "Epoch AI internal evaluation; file gpqa_diamond.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id ZU5ebpDaDPiDZLgv7cG8B9; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.017287759372442393. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "otis-mock-aime",
    "score": 0.9833333333333333,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file otis_mock_aime_2024_2025.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id 7eHNiVQCzLcAyj5LxmeeSU; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.006405884289881728. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "frontiermath",
    "score": 0.6736842105263158,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file frontiermath_tiers_1_3_v2.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id b6dXtaGSwvfRYADKYnj9Ci; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.027821981570324446. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "frontiermath-tier-4-v2",
    "score": 0.2682926829268293,
    "source": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file frontiermath_tier_4_v2.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id KansJcyeGR4m7G4p8GupDn; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.07005564203095158. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "epoch-mystery-games",
    "score": 0.43,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id K3JABAsJCbXU5YrCPH2yVW; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.0497569851956243. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "epoch-furniture",
    "score": 0.3416666666666667,
    "source": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv",
    "by": "independent",
    "date": "2026-10-08",
    "note": "Epoch AI internal evaluation; file furniture_assembly.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'deepseek-v4.1-flash_max'; run id JFQz7q3hxAKLV5pW6DmkGF; effort max; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.06000431560123094. Catalogue deepseek-v4-1-flash (deepseek-ai/DeepSeek-V4.1-Flash); Epoch model_metadata display 'DeepSeek V4.1 Flash (max)', release 2026-09-09.",
    "source_id": "epoch-internal"
  },
  {
    "model": "mistral-large-4",
    "bench": "simpleqa-verified",
    "score": 0.2,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-10-09",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-10 (zip sha256 3128af6b5c586bc13458f596201d11b618a8eea87fc1810d3eaebc70ba9dac7a, ETag \"6eaef4b906ef803c9863510b2b8646e2\"); exact model version 'mistral-large-4-0_high'; run id hvNNKdnzLvr9X8zVxWKcfk; effort high; highest published effort, latest run breaks ties; published 'Best score (across scorers)' column. Reported standard error 0.012655439943366657. Epoch label 'mistral-large-4-0_high' (display 'Mistral Large 4 (high)', release 2026-10-06) = Mistral API name mistral-large-4-0, an alias of catalogue mistral-large-4 per its identity note.",
    "source_id": "epoch-internal"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "open-persian-llm-v2-avg21",
    "score": 0.690938,
    "source": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl",
    "by": "independent",
    "date": "2026-09-19",
    "note": "Open Persian LLM Leaderboard v2 row 'google__gemini-2.0-flash-lite-001' (query id None, precision unknown, model sha unknown); Average Accuracy = mean of the 21 published task scores = 69.0938 (divided by 100). Thinking/reasoning setting not stated. No per-row run date; date is the pinned Space revision date. Identity: catalogue gemini-2-0-flash-lite-001. Row label 'google__gemini-2.0-flash-lite-001' names the stable Gemini API version gemini-2.0-flash-lite-001.",
    "source_id": "open-persian-llm-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "cptub-speakleash-avg",
    "score": 0.818735,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'gemini-2.0-flash-lite-001'; Space Average = mean(Sentiment 4.231, Language understanding 4.055, Phraseology 4.235, Tricky questions 3.854) = 4.0937 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue gemini-2-0-flash-lite-001. Stable Gemini API version gemini-2.0-flash-lite-001 (catalogue entry added after the CPTUB import).",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.726423,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "lab",
    "date": "2026-08-11",
    "note": "CPTUB entry 'speakleash/Bielik-11B-v2.3-Instruct'; Space Average = mean(Sentiment 3.974, Language understanding 3.785, Phraseology 3.550, Tricky questions 3.219) = 3.6321 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date. Identity: catalogue bielik-11b-v2-3-instruct. SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab.",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.7308,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-10-17",
    "note": "Open PT-LLM Leaderboard row 'speakleash/Bielik-11B-v2.3-Instruct' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 73.08 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha eb77dfe2d0. Identity: catalogue bielik-11b-v2-3-instruct.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "eurollm-1-7b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2982,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'utter-project/EuroLLM-1.7B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "eurollm-1-7b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.3079,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'utter-project/EuroLLM-1.7B-Instruct [bfloat16]' (source=hub, precision=bfloat16, model_sha a25c7fa65fc2a644e6270b8940dbe295b51da681); published average 30.79 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. The float16 row of the same checkpoint (31.33) is not imported; bfloat16 is the checkpoint dtype.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "eurollm-1-7b-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.352601,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'utter-project/EuroLLM-1.7B-Instruct'; Space Average = mean(Sentiment 2.244, Language understanding 1.790, Phraseology 2.260, Tricky questions 0.758) = 1.7630 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "eurollm-9b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7279,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'utter-project/EuroLLM-9B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "eurollm-9b-instruct",
    "bench": "open-pt-llm-avg9",
    "score": 0.6827,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'utter-project/EuroLLM-9B-Instruct' (source=hub, precision=bfloat16, model_sha f7ae2bc3bcbb538c0b93fa6cfbf388d9898b1ace); published average 68.27 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "eurollm-9b-instruct",
    "bench": "cptub-speakleash-avg",
    "score": 0.629129,
    "source": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json",
    "by": "independent",
    "date": "2026-08-11",
    "note": "CPTUB entry 'utter-project/EuroLLM-9B-Instruct'; Space Average = mean(Sentiment 3.365, Language understanding 3.300, Phraseology 3.170, Tricky questions 2.747) = 3.1456 on a 0-5 judge scale, divided by 5. No per-row run date; date is the pinned Space revision date.",
    "source_id": "speakleash-cptub"
  },
  {
    "model": "eurollm-9b-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.5475,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2025-03-02",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'EuroLLM-9B-Instruct', model path utter-project/EuroLLM-9B-Instruct; eq-bench_v2_pl score 54.75 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-03-02 16:41:02.",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "eurollm-22b-instruct-2512",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8771,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'utter-project/EuroLLM-22B-Instruct-2512', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "eurollm-22b-instruct-2512",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7148,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'utter-project/EuroLLM-22B-Instruct-2512', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "apertus-8b-instruct-2509",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8035,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'swiss-ai/Apertus-8B-Instruct-2509', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "apertus-8b-instruct-2509",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5807,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'swiss-ai/Apertus-8B-Instruct-2509', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "apertus-70b-instruct-2509",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8818,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'swiss-ai/Apertus-70B-Instruct-2509', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "apertus-70b-instruct-2509",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7611,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'swiss-ai/Apertus-70B-Instruct-2509', instruction-tuned (📝), default setting; EuroEval version 15.16.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "apertus-8b-instruct-2509",
    "bench": "mideind-icelandic-average",
    "score": 0.077325,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Apertus 8B Instruct (2509)' (hosted_vllm/swiss-ai/Apertus-8B-Instruct-2509), thinking='none', gen_kwargs={\"max_tokens\": 32000}. Published 'average' (Knowledge 0.042785, Language Proficiency 0.10323). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at.",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "apertus-1-5-8b",
    "bench": "mideind-icelandic-average",
    "score": 0.116917,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Apertus 1.5 8B' (hosted_vllm/swiss-ai/Apertus-v1.5-8B), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"chat_template_kwargs\": {\"enable_thinking\": true}}}. Published 'average' (Knowledge 0.062538, Language Proficiency 0.1577). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at. Thinking mode on (the only Apertus 1.5 setting listed).",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "apertus-1-5-70b",
    "bench": "mideind-icelandic-average",
    "score": 0.175296,
    "source": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/7b12ee6799a4735154be85170c53d7266b2de4e6/leaderboard.json",
    "by": "independent",
    "date": "2026-09-22",
    "note": "Miðeind Icelandic LLM Leaderboard entry 'Apertus 1.5 70B' (hosted_vllm/swiss-ai/Apertus-v1.5-70B), thinking='on', gen_kwargs={\"max_tokens\": 32000, \"extra_body\": {\"chat_template_kwargs\": {\"enable_thinking\": true}}}. Published 'average' (Knowledge 0.086421, Language Proficiency 0.241952). Board generated 2026-09-30T09:19:24Z; Space revision 7b12ee6799a4735154be85170c53d7266b2de4e6. Date = latest benchmark_evaluated_at. Thinking mode on (the only Apertus 1.5 setting listed).",
    "source_id": "mideind-icelandic-llm-leaderboard"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mmlu",
    "score": 0.6743,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MMLU' (5-shot), section 'General Tasks' = 67.43; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mmlu-pro",
    "score": 0.4348,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 43.48; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "bbh",
    "score": 0.6936,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'BBH' (3-shot, CoT), section 'General Tasks' = 69.36; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "gpqa-unspecified",
    "score": 0.3215,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 32.15; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "ifeval-instruction-strict",
    "score": 0.8694,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 86.94; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "ifeval-prompt-strict",
    "score": 0.8171,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 81.71; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "gsm8k",
    "score": 0.8135,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'GSM8K' (8-shot), section 'Math Tasks' = 81.35; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "gsm-symbolic",
    "score": 0.775,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'GSM8K Symbolic' (8-shot), section 'Math Tasks' = 77.5; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "minerva-math",
    "score": 0.6644,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 66.44; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "deepmind-math",
    "score": 0.4383,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 43.83; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "humaneval",
    "score": 0.81,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'HumanEval' (pass@1), section 'Code Tasks' = 81; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "humaneval-plus",
    "score": 0.75,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'HumanEval+' (pass@1), section 'Code Tasks' = 75; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mbpp",
    "score": 0.73,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MBPP' (pass@1), section 'Code Tasks' = 73; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mbpp-plus",
    "score": 0.64,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MBPP+' (pass@1), section 'Code Tasks' = 64; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "cruxeval-o",
    "score": 0.4125,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 41.25; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "bigcodebench",
    "score": 0.379,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 37.9; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "bfcl-v3",
    "score": 0.5756,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 57.56; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "multipl-e",
    "score": 0.4946,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 49.46; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mmmlu",
    "score": 0.5519,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 55.19; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "include",
    "score": 0.5051,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 50.51; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "mgsm-5lang",
    "score": 0.4448,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Micro Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 44.48; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mmlu",
    "score": 0.6865,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MMLU' (5-shot), section 'General Tasks' = 68.65; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mmlu-pro",
    "score": 0.4494,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 44.94; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "bbh",
    "score": 0.6634,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'BBH' (3-shot, CoT), section 'General Tasks' = 66.34; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "gpqa-unspecified",
    "score": 0.3259,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 32.59; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "ifeval-instruction-strict",
    "score": 0.8478,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 84.78; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "ifeval-prompt-strict",
    "score": 0.781,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 78.1; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "gsm8k",
    "score": 0.8469,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'GSM8K' (8-shot), section 'Math Tasks' = 84.69; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "gsm-symbolic",
    "score": 0.811,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'GSM8K Symbolic' (8-shot), section 'Math Tasks' = 81.1; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "minerva-math",
    "score": 0.6964,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 69.64; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "deepmind-math",
    "score": 0.4992,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 49.92; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "humaneval",
    "score": 0.83,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'HumanEval' (pass@1), section 'Code Tasks' = 83; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "humaneval-plus",
    "score": 0.76,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'HumanEval+' (pass@1), section 'Code Tasks' = 76; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mbpp",
    "score": 0.8,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MBPP' (pass@1), section 'Code Tasks' = 80; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mbpp-plus",
    "score": 0.69,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MBPP+' (pass@1), section 'Code Tasks' = 69; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "cruxeval-o",
    "score": 0.3963,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 39.63; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "bigcodebench",
    "score": 0.4106,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 41.06; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "bfcl-v3",
    "score": 0.5765,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 57.65; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "multipl-e",
    "score": 0.5583,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 55.83; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mmmlu",
    "score": 0.6187,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 61.87; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "include",
    "score": 0.5312,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 53.12; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "mgsm-5lang",
    "score": 0.4536,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md",
    "by": "lab",
    "date": "2025-11-03",
    "note": "IBM own evaluation in the pinned granite-4.0-h-small model card (Apache-2.0), Evaluation Results table (columns Micro Dense / H Micro Dense / H Tiny MoE / H Small MoE; the same table is printed in the h-micro and h-tiny cards). Same table, row mapping and exclusions as the existing granite-4-0-h-small / granite-4-0-micro rows: MMMLU 5-shot over 11 languages, INCLUDE 5-shot over 14, MGSM 8-shot over en/es/fr/ja/zh; shots/CoT per the card's Metric column; AGI EVAL, AlpacaEval 2.0, ArenaHard, IFEval Average and safety rows not used. Column 'H Tiny MoE', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 45.36; divided by 100.",
    "source_id": "card2-ibm-granite-granite-4-0-h-small"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mmlu",
    "score": 0.3501,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MMLU' (5-shot), section 'General Tasks' = 35.01; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mmlu-pro",
    "score": 0.1213,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 12.13; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "bbh",
    "score": 0.3307,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'BBH' (3-shot, CoT), section 'General Tasks' = 33.07; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "gpqa-unspecified",
    "score": 0.2411,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 24.11; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "ifeval-instruction-strict",
    "score": 0.6163,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 61.63; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "ifeval-prompt-strict",
    "score": 0.4917,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 49.17; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "gsm8k",
    "score": 0.3071,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'GSM8K' (8-shot), section 'Math Tasks' = 30.71; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "gsm-symbolic",
    "score": 0.2676,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 26.76; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "minerva-math",
    "score": 0.1304,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 13.04; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "deepmind-math",
    "score": 0.0845,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 8.45; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "humaneval",
    "score": 0.39,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'HumanEval' (pass@1), section 'Code Tasks' = 39; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "humaneval-plus",
    "score": 0.37,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'HumanEval+' (pass@1), section 'Code Tasks' = 37; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mbpp",
    "score": 0.48,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MBPP' (pass@1), section 'Code Tasks' = 48; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mbpp-plus",
    "score": 0.38,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MBPP+' (pass@1), section 'Code Tasks' = 38; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "cruxeval-o",
    "score": 0.2375,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 23.75; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "bigcodebench",
    "score": 0.1114,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 11.14; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "bfcl-v3",
    "score": 0.3932,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 39.32; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "multipl-e",
    "score": 0.1599,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 15.99; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mmmlu",
    "score": 0.2823,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 28.23; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "include",
    "score": 0.2774,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 27.74; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-350m",
    "bench": "mgsm-5lang",
    "score": 0.1472,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '350M Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 14.72; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mmlu",
    "score": 0.3621,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MMLU' (5-shot), section 'General Tasks' = 36.21; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mmlu-pro",
    "score": 0.1438,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 14.38; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "bbh",
    "score": 0.3328,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'BBH' (3-shot, CoT), section 'General Tasks' = 33.28; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "gpqa-unspecified",
    "score": 0.2612,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 26.12; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "ifeval-instruction-strict",
    "score": 0.6763,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 67.63; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "ifeval-prompt-strict",
    "score": 0.5564,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 55.64; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "gsm8k",
    "score": 0.3927,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'GSM8K' (8-shot), section 'Math Tasks' = 39.27; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "gsm-symbolic",
    "score": 0.337,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 33.7; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "minerva-math",
    "score": 0.0576,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 5.76; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "deepmind-math",
    "score": 0.062,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 6.2; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "humaneval",
    "score": 0.38,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'HumanEval' (pass@1), section 'Code Tasks' = 38; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "humaneval-plus",
    "score": 0.35,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'HumanEval+' (pass@1), section 'Code Tasks' = 35; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mbpp",
    "score": 0.49,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MBPP' (pass@1), section 'Code Tasks' = 49; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mbpp-plus",
    "score": 0.44,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MBPP+' (pass@1), section 'Code Tasks' = 44; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "cruxeval-o",
    "score": 0.255,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 25.5; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "bigcodebench",
    "score": 0.1123,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 11.23; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "bfcl-v3",
    "score": 0.4332,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 43.32; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "multipl-e",
    "score": 0.1431,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 14.31; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mmmlu",
    "score": 0.2795,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 27.95; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "include",
    "score": 0.2709,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 27.09; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-350m",
    "bench": "mgsm-5lang",
    "score": 0.1616,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 350M Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 16.16; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mmlu",
    "score": 0.5939,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MMLU' (5-shot), section 'General Tasks' = 59.39; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mmlu-pro",
    "score": 0.3402,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 34.02; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "bbh",
    "score": 0.6037,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'BBH' (3-shot, CoT), section 'General Tasks' = 60.37; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gpqa-unspecified",
    "score": 0.2991,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 29.91; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "ifeval-instruction-strict",
    "score": 0.8082,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 80.82; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "ifeval-prompt-strict",
    "score": 0.7394,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 73.94; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gsm8k",
    "score": 0.7635,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'GSM8K' (8-shot), section 'Math Tasks' = 76.35; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "gsm-symbolic",
    "score": 0.723,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 72.3; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "minerva-math",
    "score": 0.4528,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 45.28; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "deepmind-math",
    "score": 0.34,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 34; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "humaneval",
    "score": 0.74,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'HumanEval' (pass@1), section 'Code Tasks' = 74; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "humaneval-plus",
    "score": 0.69,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'HumanEval+' (pass@1), section 'Code Tasks' = 69; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mbpp",
    "score": 0.65,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MBPP' (pass@1), section 'Code Tasks' = 65; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mbpp-plus",
    "score": 0.57,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MBPP+' (pass@1), section 'Code Tasks' = 57; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "cruxeval-o",
    "score": 0.3313,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 33.13; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "bigcodebench",
    "score": 0.3018,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 30.18; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "bfcl-v3",
    "score": 0.5482,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 54.82; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "multipl-e",
    "score": 0.3224,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 32.24; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mmmlu",
    "score": 0.45,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 45; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "include",
    "score": 0.4212,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 42.12; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-1b",
    "bench": "mgsm-5lang",
    "score": 0.3784,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column '1B Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 37.84; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mmlu",
    "score": 0.5974,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MMLU' (5-shot), section 'General Tasks' = 59.74; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mmlu-pro",
    "score": 0.3286,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MMLU-Pro' (5-shot, CoT), section 'General Tasks' = 32.86; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "bbh",
    "score": 0.5968,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'BBH' (3-shot, CoT), section 'General Tasks' = 59.68; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "gpqa-unspecified",
    "score": 0.2969,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'GPQA' (0-shot, CoT), section 'General Tasks' = 29.69; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "ifeval-instruction-strict",
    "score": 0.8237,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'IFEval' (Instruct, Strict), section 'Alignment Tasks' = 82.37; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "ifeval-prompt-strict",
    "score": 0.7468,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'IFEval' (Prompt, Strict), section 'Alignment Tasks' = 74.68; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "gsm8k",
    "score": 0.6983,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'GSM8K' (8-shot), section 'Math Tasks' = 69.83; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "gsm-symbolic",
    "score": 0.6572,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'GSM Symbolic' (8-shot), section 'Math Tasks' = 65.72; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "minerva-math",
    "score": 0.494,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'Minerva Math' (0-shot, CoT), section 'Math Tasks' = 49.4; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "deepmind-math",
    "score": 0.3498,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'DeepMind Math' (0-shot, CoT), section 'Math Tasks' = 34.98; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "humaneval",
    "score": 0.73,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'HumanEval' (pass@1), section 'Code Tasks' = 73; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "humaneval-plus",
    "score": 0.68,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'HumanEval+' (pass@1), section 'Code Tasks' = 68; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mbpp",
    "score": 0.69,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MBPP' (pass@1), section 'Code Tasks' = 69; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mbpp-plus",
    "score": 0.6,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MBPP+' (pass@1), section 'Code Tasks' = 60; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "cruxeval-o",
    "score": 0.36,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'CRUXEval-O' (pass@1), section 'Code Tasks' = 36; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "bigcodebench",
    "score": 0.2912,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'BigCodeBench' (pass@1), section 'Code Tasks' = 29.12; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "bfcl-v3",
    "score": 0.5021,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'BFCL v3' (no metric printed), section 'Tool Calling Tasks' = 50.21; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "multipl-e",
    "score": 0.3611,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MULTIPLE' (pass@1), section 'Multilingual Tasks' = 36.11; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mmmlu",
    "score": 0.4943,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MMMLU' (5-shot), section 'Multilingual Tasks' = 49.43; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "include",
    "score": 0.4335,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'INCLUDE' (5-shot), section 'Multilingual Tasks' = 43.35; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mgsm-5lang",
    "score": 0.2752,
    "source": "https://huggingface.co/ibm-granite/granite-4.0-h-1b/blob/d18cca4c121edb87d022116d281ce212c9136f57/README.md",
    "by": "lab",
    "date": "2025-10-23",
    "note": "IBM own evaluation in the pinned granite-4.0-h-1b model card (Apache-2.0), Evaluation Results table (columns 350M Dense / H 350M Dense / 1B Dense / H 1B Dense = granite-4.0-350m, granite-4.0-h-350m, granite-4.0-1b, granite-4.0-h-1b; identical table in the granite-4.0-1b card). Row mapping and exclusions as for the existing Granite 4.0 card rows (MMMLU 11 languages, INCLUDE 14, MGSM en/es/fr/ja/zh; AGI EVAL, IFEval Average and safety rows not used). Column 'H 1B Dense', row 'MGSM' (8-shot), section 'Multilingual Tasks' = 27.52; divided by 100.",
    "source_id": "card3-ibm-granite-granite-4-0-h-1b"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "arc-challenge",
    "score": 0.6262,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'ARC-Challenge' = 62.62; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "hellaswag",
    "score": 0.8448,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'Hellaswag' = 84.48; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "mmlu",
    "score": 0.6534,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'MMLU' = 65.34; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "winogrande",
    "score": 0.7537,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'Winogrande' = 75.37; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "gsm8k",
    "score": 0.7384,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'GSM8K' = 73.84; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "arc-challenge",
    "score": 0.5461,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'ARC-Challenge' = 54.61; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "hellaswag",
    "score": 0.7514,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'Hellaswag' = 75.14; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "mmlu",
    "score": 0.5531,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'MMLU' = 55.31; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "winogrande",
    "score": 0.6748,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'Winogrande' = 67.48; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "gsm8k",
    "score": 0.5276,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'GSM8K' = 52.76; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "arc-challenge",
    "score": 0.5042,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'ARC-Challenge' = 50.42; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "hellaswag",
    "score": 0.7301,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'Hellaswag' = 73.01; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "mmlu",
    "score": 0.5219,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'MMLU' = 52.19; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "winogrande",
    "score": 0.6487,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'Winogrande' = 64.87; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "gsm8k",
    "score": 0.4897,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'GSM8K' = 48.97; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "arc-challenge",
    "score": 0.4266,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'ARC-Challenge' = 42.66; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "hellaswag",
    "score": 0.6597,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'Hellaswag' = 65.97; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "mmlu",
    "score": 0.2613,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'MMLU' = 26.13; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "winogrande",
    "score": 0.6235,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'Winogrande' = 62.35; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "gsm8k",
    "score": 0.3388,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V1' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'GSM8K' = 33.88; divided by 100. TruthfulQA column and the Avg not used (variant not printed). Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-8b",
    "bench": "ifeval-unspecified",
    "score": 0.7208,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V2' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-8B-Instruct', column 'IFEval' = 72.08; divided by 100. Only the IFEval column is used: the card's BBH, MATH Lvl 5, GPQA, MUSR and MMLU-Pro columns are the V2 leaderboard's normalised scores (e.g. GPQA 0 for 1B-A400M), not raw accuracies. Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-2b",
    "bench": "ifeval-unspecified",
    "score": 0.6286,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V2' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-2B-Instruct', column 'IFEval' = 62.86; divided by 100. Only the IFEval column is used: the card's BBH, MATH Lvl 5, GPQA, MUSR and MMLU-Pro columns are the V2 leaderboard's normalised scores (e.g. GPQA 0 for 1B-A400M), not raw accuracies. Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-3b-a800m",
    "bench": "ifeval-unspecified",
    "score": 0.5516,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V2' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-3B-A800M-Instruct', column 'IFEval' = 55.16; divided by 100. Only the IFEval column is used: the card's BBH, MATH Lvl 5, GPQA, MUSR and MMLU-Pro columns are the V2 leaderboard's normalised scores (e.g. GPQA 0 for 1B-A400M), not raw accuracies. Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-1-1b-a400m",
    "bench": "ifeval-unspecified",
    "score": 0.4686,
    "source": "https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct/blob/a02780686e08a03fe0d2679a293b5c74a90efa89/README.md",
    "by": "lab",
    "date": "2025-01-31",
    "note": "IBM own evaluation in the pinned granite-3.1-3b-a800m-instruct model card (Apache-2.0), Evaluation Results table captioned 'HuggingFace Open LLM Leaderboard V2' (IBM's run of that leaderboard's task suite on its Granite 3.1 instruct models: 8B, 2B, 3B-A800M, 1B-A400M; identical table in the 1b-a400m card), row 'Granite-3.1-1B-A400M-Instruct', column 'IFEval' = 46.86; divided by 100. Only the IFEval column is used: the card's BBH, MATH Lvl 5, GPQA, MUSR and MMLU-Pro columns are the V2 leaderboard's normalised scores (e.g. GPQA 0 for 1B-A400M), not raw accuracies. Card 'Release Date: December 18th, 2024'; date = pinned card revision.",
    "source_id": "card3-ibm-granite-granite-3-1-3b-a800m-instruct"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "ifeval-unspecified",
    "score": 0.3239,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'IFEval' (0-shot), column 'Granite-3.0 (400M active / 1B total)' = 32.39; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifeval-unspecified",
    "score": 0.4168,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'IFEval' (0-shot), column 'Llama-3.2 1B' = 41.68; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "ifeval-unspecified",
    "score": 0.4249,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'IFEval' (0-shot), column 'Granite-3.0 (800M active / 3B total)' = 42.49; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "mmlu",
    "score": 0.32,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU' (5-shot), column 'Granite-3.0 (400M active / 1B total)' = 32.00; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmlu",
    "score": 0.454,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU' (5-shot), column 'Llama-3.2 1B' = 45.40; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "mmlu",
    "score": 0.5016,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU' (5-shot), column 'Granite-3.0 (800M active / 3B total)' = 50.16; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "mmlu-pro",
    "score": 0.1221,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU-Pro' (5-shot), column 'Granite-3.0 (400M active / 1B total)' = 12.21; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmlu-pro",
    "score": 0.1952,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU-Pro' (5-shot), column 'Llama-3.2 1B' = 19.52; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "mmlu-pro",
    "score": 0.2051,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MMLU-Pro' (5-shot), column 'Granite-3.0 (800M active / 3B total)' = 20.51; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "openbookqa",
    "score": 0.384,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'OBQA' (0-shot), column 'Granite-3.0 (400M active / 1B total)' = 38.40; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "openbookqa",
    "score": 0.346,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'OBQA' (0-shot), column 'Llama-3.2 1B' = 34.60; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "openbookqa",
    "score": 0.408,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'OBQA' (0-shot), column 'Granite-3.0 (800M active / 3B total)' = 40.80; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "social-iqa",
    "score": 0.4755,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'SIQA' (0-shot), column 'Granite-3.0 (400M active / 1B total)' = 47.55; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "social-iqa",
    "score": 0.355,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'SIQA' (0-shot), column 'Llama-3.2 1B' = 35.50; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "social-iqa",
    "score": 0.5995,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'SIQA' (0-shot), column 'Granite-3.0 (800M active / 3B total)' = 59.95; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "hellaswag",
    "score": 0.6559,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'Hellaswag' (10-shot), column 'Granite-3.0 (400M active / 1B total)' = 65.59; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "hellaswag",
    "score": 0.5974,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'Hellaswag' (10-shot), column 'Llama-3.2 1B' = 59.74; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "hellaswag",
    "score": 0.7186,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'Hellaswag' (10-shot), column 'Granite-3.0 (800M active / 3B total)' = 71.86; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "winogrande",
    "score": 0.6117,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'WinoGrande' (5-shot), column 'Granite-3.0 (400M active / 1B total)' = 61.17; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "winogrande",
    "score": 0.6101,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'WinoGrande' (5-shot), column 'Llama-3.2 1B' = 61.01; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "winogrande",
    "score": 0.6701,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'WinoGrande' (5-shot), column 'Granite-3.0 (800M active / 3B total)' = 67.01; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "boolq",
    "score": 0.7012,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BoolQ' (5-shot), column 'Granite-3.0 (400M active / 1B total)' = 70.12; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "boolq",
    "score": 0.6673,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BoolQ' (5-shot), column 'Llama-3.2 1B' = 66.73; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "boolq",
    "score": 0.7865,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BoolQ' (5-shot), column 'Granite-3.0 (800M active / 3B total)' = 78.65; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "arc-challenge",
    "score": 0.4121,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'ARC-C' (25-shot), column 'Granite-3.0 (400M active / 1B total)' = 41.21; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "arc-challenge",
    "score": 0.4138,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'ARC-C' (25-shot), column 'Llama-3.2 1B' = 41.38; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "arc-challenge",
    "score": 0.5094,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'ARC-C' (25-shot), column 'Granite-3.0 (800M active / 3B total)' = 50.94; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "gpqa-unspecified",
    "score": 0.2307,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GPQA' (0-shot), column 'Granite-3.0 (400M active / 1B total)' = 23.07; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gpqa-unspecified",
    "score": 0.2567,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GPQA' (0-shot), column 'Llama-3.2 1B' = 25.67; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "gpqa-unspecified",
    "score": 0.2685,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GPQA' (0-shot), column 'Granite-3.0 (800M active / 3B total)' = 26.85; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "bbh",
    "score": 0.3177,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BBH' (3-shot), column 'Granite-3.0 (400M active / 1B total)' = 31.77; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "bbh",
    "score": 0.3354,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BBH' (3-shot), column 'Llama-3.2 1B' = 33.54; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "bbh",
    "score": 0.377,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'BBH' (3-shot), column 'Granite-3.0 (800M active / 3B total)' = 37.70; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "humaneval",
    "score": 0.3018,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'HumanEvalSynthesis' (pass@1), column 'Granite-3.0 (400M active / 1B total)' = 30.18; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "humaneval",
    "score": 0.3598,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'HumanEvalSynthesis' (pass@1), column 'Llama-3.2 1B' = 35.98; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "humaneval",
    "score": 0.3963,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'HumanEvalSynthesis' (pass@1), column 'Granite-3.0 (800M active / 3B total)' = 39.63; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "mbpp",
    "score": 0.154,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MBPP' (no metric printed in Table 11; pass@1 in Table 10), column 'Granite-3.0 (400M active / 1B total)' = 15.40; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mbpp",
    "score": 0.37,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MBPP' (no metric printed in Table 11; pass@1 in Table 10), column 'Llama-3.2 1B' = 37.00; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "mbpp",
    "score": 0.274,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MBPP' (no metric printed in Table 11; pass@1 in Table 10), column 'Granite-3.0 (800M active / 3B total)' = 27.40; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "gsm8k",
    "score": 0.2631,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GSM8k' (5-shot,cot), column 'Granite-3.0 (400M active / 1B total)' = 26.31; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gsm8k",
    "score": 0.2616,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GSM8k' (5-shot,cot), column 'Llama-3.2 1B' = 26.16; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "gsm8k",
    "score": 0.4754,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'GSM8k' (5-shot,cot), column 'Granite-3.0 (800M active / 3B total)' = 47.54; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "math-subset-unspecified",
    "score": 0.1088,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MATH' (4-shot), column 'Granite-3.0 (400M active / 1B total)' = 10.88; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "math-subset-unspecified",
    "score": 0.1762,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "independent",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MATH' (4-shot), column 'Llama-3.2 1B' = 17.62; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "math-subset-unspecified",
    "score": 0.1986,
    "source": "https://github.com/ibm-granite/granite-3.0-language-models/blob/5960b7fb3308588bf5f86348278ec9d6015ab647/paper.pdf",
    "by": "lab",
    "date": "2024-11-09",
    "note": "IBM Granite 3.0 technical report (paper.pdf in the Apache-2.0 ibm-granite/granite-3.0-language-models repository @5960b7f), Table 11 'MoE Instruction Models', row 'MATH' (4-shot), column 'Granite-3.0 (800M active / 3B total)' = 19.86; divided by 100. Same harness, settings and row exclusions as the existing Table 10 rows of this source (Section 7: baseline 'models were also evaluated by ourselves using LM evaluation harness ... using the same pipeline and configuration'; all columns instruct models). SmolLM 360M / 1.7B columns not used (SmolLM instruct version v0.1/v0.2 not stated). Date is the last commit of the pinned paper.pdf.",
    "source_id": "granite-3-0-paper-github"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.816,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-micro', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6768,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-micro', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8121,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-tiny', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7043,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-4.0-h-tiny', instruction-tuned (📝), default setting; EuroEval version 16.8.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-3-0-1b-a400m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.348,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3.0-1b-a400m-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-3-0-3b-a800m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5582,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'ibm-granite/granite-3.0-3b-a800m-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "granite-4-0-h-micro",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.916,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-4.0-h-micro': factual consistency rate 91.6% (hallucination rate 8.4%), answer rate 100.0%, average summary length 71.8 words. HHEM-2.3 judge. Identity: IBM's Granite 4.0 H Micro (ibm-granite/granite-4.0-h-micro); the README groups it with the other Granite 4.0 rows.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "granite-4-0-h-tiny",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.927,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'IBM Granite-4.0-h-tiny': factual consistency rate 92.7% (hallucination rate 7.3%), answer rate 99.9%, average summary length 94.7 words. HHEM-2.3 judge. Identity: IBM's Granite 4.0 H Tiny (ibm-granite/granite-4.0-h-tiny).",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "exaone-3-5-2-4b",
    "bench": "arena-hard-unspecified",
    "score": 0.482,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct/blob/ccce25bd39c141fe053e0bc75818a8f5fe962802/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-2.4B-Instruct model card (pinned ccce25b), Evaluation table, row 'EXAONE 3.5 2.4B', column 'Arena-Hard' printed 48.2 (version/judge not stated; the technical report is the source of the card table). Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-3-5-7-8b",
    "bench": "ifeval-unspecified",
    "score": 0.789,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct/blob/553ea250b9a5317231459279d5847d6cf955b9aa/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-7.8B-Instruct model card (pinned 553ea25), Evaluation table, row 'EXAONE 3.5 7.8B', column 'IFEval' printed 78.9 (aggregation not stated). Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-3-5-7-8b",
    "bench": "arena-hard-unspecified",
    "score": 0.687,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct/blob/553ea250b9a5317231459279d5847d6cf955b9aa/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-7.8B-Instruct model card (pinned 553ea25), Evaluation table, row 'EXAONE 3.5 7.8B', column 'Arena-Hard' printed 68.7 (version/judge not stated; the technical report is the source of the card table). Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-3-5-32b",
    "bench": "ifeval-unspecified",
    "score": 0.817,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-32B-Instruct/blob/3d27aab477c042df2e174d36eea47b77c8871ef0/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-32B-Instruct model card (pinned 3d27aab), Evaluation table, row 'EXAONE 3.5 32B', column 'IFEval' printed 81.7 (aggregation not stated). Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-3-5-32b",
    "bench": "arena-hard-unspecified",
    "score": 0.786,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-32B-Instruct/blob/3d27aab477c042df2e174d36eea47b77c8871ef0/README.md#evaluation",
    "by": "lab",
    "date": "2024-12-01",
    "note": "LG AI Research EXAONE-3.5-32B-Instruct model card (pinned 3d27aab), Evaluation table, row 'EXAONE 3.5 32B', column 'Arena-Hard' printed 78.6 (version/judge not stated; the technical report is the source of the card table). Date is the model repository creation date. Vendor-reported (rule b): LG AI Research's own figure for its own model, hand-recorded one figure at a time; other labs' columns not used. EXAONE model licence is not an open licence, so no licensed copy is used.",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-32b",
    "bench": "math-500",
    "score": 0.957,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 32B', column 'MATH-500 (pass@1)' = 95.7. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-32b",
    "bench": "aime-2024",
    "score": 0.721,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 32B', column 'AIME 2024 (pass@1 / cons@64)' pass@1 value' = 72.1. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-32b",
    "bench": "aime-2025",
    "score": 0.658,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 32B', column 'AIME 2025 (pass@1 / cons@64)' pass@1 value' = 65.8. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-32b",
    "bench": "gpqa-diamond",
    "score": 0.661,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 32B', column 'GPQA Diamond (pass@1)' = 66.1. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-32b",
    "bench": "livecodebench-unversioned",
    "score": 0.595,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 32B', column 'Live Code Bench (pass@1)' = 59.5. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-7-8b",
    "bench": "math-500",
    "score": 0.948,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 7.8B', column 'MATH-500 (pass@1)' = 94.8. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-7-8b",
    "bench": "aime-2024",
    "score": 0.7,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 7.8B', column 'AIME 2024 (pass@1 / cons@64)' pass@1 value' = 70.0. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-7-8b",
    "bench": "aime-2025",
    "score": 0.596,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 7.8B', column 'AIME 2025 (pass@1 / cons@64)' pass@1 value' = 59.6. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-7-8b",
    "bench": "gpqa-diamond",
    "score": 0.626,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 7.8B', column 'GPQA Diamond (pass@1)' = 62.6. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-7-8b",
    "bench": "livecodebench-unversioned",
    "score": 0.552,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 7.8B', column 'Live Code Bench (pass@1)' = 55.2. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-2-4b",
    "bench": "math-500",
    "score": 0.923,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 2.4B', column 'MATH-500 (pass@1)' = 92.3. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-2-4b",
    "bench": "aime-2024",
    "score": 0.525,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 2.4B', column 'AIME 2024 (pass@1 / cons@64)' pass@1 value' = 52.5. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-2-4b",
    "bench": "aime-2025",
    "score": 0.479,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 2.4B', column 'AIME 2025 (pass@1 / cons@64)' pass@1 value' = 47.9. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-2-4b",
    "bench": "gpqa-diamond",
    "score": 0.543,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 2.4B', column 'GPQA Diamond (pass@1)' = 54.3. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "exaone-deep-2-4b",
    "bench": "livecodebench-unversioned",
    "score": 0.466,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-Deep-32B/blob/4e9a4c20b6c3f9093777bfe6a98b2be508abb919/README.md#evaluation",
    "by": "lab",
    "date": "2025-03-12",
    "note": "LG AI Research EXAONE-Deep-32B model card (pinned 4e9a4c2), Evaluation table (one table for the three EXAONE Deep sizes), row 'EXAONE Deep 2.4B', column 'Live Code Bench (pass@1)' = 46.6. LiveCodeBench window not stated in the card. Date is the repository creation date. Vendor-reported (rule b), hand-recorded; DeepSeek/QwQ/o1-mini rows not used (EXAONE licence is not open).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "alpacaeval1-winrate",
    "score": 0.4571,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'falcon-7b-instruct' (mode 'verified'), win_rate = 45.71 (standard error 1.75); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 478.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.0404,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'falcon-7b-instruct' (mode 'verified'), length_controlled_winrate = 4.04 (LC standard error not listed); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 2.15; average length 478.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.701,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'TII falcon-7B-instruct': factual consistency rate 70.1% (hallucination rate 29.9%), answer rate 90.0%, average summary length 75.5 words. HHEM-2.3 judge. README: 'tiiuae/falcon-7b-instruct ... accessed via Hugging Face's API'.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "falcon-40b-instruct",
    "bench": "alpacaeval1-winrate",
    "score": 0.4571,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 1.0 leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), alpaca_eval_gpt4_leaderboard.csv, row 'falcon-40b-instruct' (mode 'verified'), win_rate = 45.71 (standard error 1.75); stored /100. 805 AlpacaEval instructions judged by alpaca_eval_gpt4 against text_davinci_003 (the reference, 50.00 by definition); average length 662.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "falcon-40b-instruct",
    "bench": "alpacaeval2-lc",
    "score": 0.0561,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'falcon-40b-instruct' (mode 'verified'), length_controlled_winrate = 5.61 (LC standard error not listed); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 3.34; average length 662.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "falcon-40b-instruct",
    "bench": "mt-bench-gpt4-single",
    "score": 5.17,
    "source": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv",
    "by": "independent",
    "date": "2025-08-04",
    "note": "LMSYS Chatbot Arena leaderboard table (arena-leaderboard Space @6f2eeeb, file leaderboard_table_20250804.csv), row key 'falcon-40b-instruct' ('Falcon-40B-Instruct', TII, link https://huggingface.co/tiiuae/falcon-40b-instruct): 'MT-bench (score)' = 5.17 (GPT-4 single-answer grading, 1-10 scale, mean over 80 two-turn questions). FastChat monitor_md.py (@587d5cf): 'The MT-bench scores (single-answer grading on a scale of 10) are computed by fastchat.llm_judge'. Date = the pinned table's file date (first publication not traced).",
    "source_id": "lmsys-arena-space-mtbench"
  },
  {
    "model": "falcon2-11b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7373,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'tiiuae/falcon-11B', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "falcon2-11b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6417,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-13",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/falcon-11B' (source=hub, precision=bfloat16, model_sha 066e3bf4e2d9aaeefa129af0a6d39727d27816b3); published average 64.17 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision 066e3bf has the same weight files as the current main (a4ab868).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "medgemma-4b-it",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.2942,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'medgemma-4b-it'; 'Average Performance' 29.42 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "medgemma-4b-it",
    "bench": "open-pt-llm-avg9",
    "score": 0.6824,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-15",
    "note": "Open PT-LLM Leaderboard row 'google/medgemma-4b-it' (source=hub, precision=bfloat16, model_sha efe6cc02361759b6bd501c654ddb7c9d25ec509d); published average 68.24 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main (gated repo).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "medgemma-27b-it",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.408,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-06-03",
    "note": "BRIDGE Zero-Shot leaderboard row 'medgemma-27b-it'; 'Average Performance' 40.8 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "medgemma-27b-it",
    "bench": "open-pt-llm-avg9",
    "score": 0.7746,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-15",
    "note": "Open PT-LLM Leaderboard row 'google/medgemma-27b-it' (source=hub, precision=bfloat16, model_sha 2d3e00ea38b50018bf5dd3aa1009457cd2d5a48f); published average 77.46 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main (gated repo).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "medgemma-27b-text-it",
    "bench": "bridge-clinical-zeroshot-avg",
    "score": 0.4047,
    "source": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json",
    "by": "independent",
    "date": "2025-12-07",
    "note": "BRIDGE Zero-Shot leaderboard row 'medgemma-27b-text-it'; 'Average Performance' 40.47 over all 87 tasks, divided by 100. Date = leaderboard update in which the model was added (models-log.md). Space revision 579abc27d8659fce3ca357ee9108d4e279fbeb38.",
    "source_id": "bridge-medical-leaderboard"
  },
  {
    "model": "medgemma-27b-text-it",
    "bench": "open-pt-llm-avg9",
    "score": 0.7946,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-07-15",
    "note": "Open PT-LLM Leaderboard row 'google/medgemma-27b-text-it' (source=hub, precision=bfloat16, model_sha 6b08c481126ff65a9b8fa5ab4d691b152b8edb5d); published average 79.46 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main (gated repo).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "nexusraven-v2-13b",
    "bench": "bfcl-v3",
    "score": 0.3599,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Nexusflow-Raven-v2 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 35.99%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/Nexusflow/NexusRaven-V2-13B; Organization Nexusflow, License Apache 2.0. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "firefunction-v2",
    "bench": "bfcl-v3",
    "score": 0.5235,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'FireFunction-v2 (FC)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.35%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/fireworks-ai/firefunction-v2; Organization Fireworks, License Apache 2.0. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "sky-t1-32b-preview",
    "bench": "bfcl-v3",
    "score": 0.5241,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-06-14/score/data_overall.csv",
    "by": "independent",
    "date": "2025-06-14",
    "note": "Published entry 'Sky-T1-32B-Preview (Prompt)'. BFCL v3 leaderboard snapshot 2025-06-14 in the BFCL-Result archive (folder named after the leaderboard update date); field Overall Acc = 52.41%. v3 category set (Non-Live, Live, Multi-Turn, relevance/irrelevance; no v4 agentic web-search/memory categories). Native function calling (FC) selected when available, otherwise Prompt; no component scores. Model Link https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview; Organization NovaSky-AI, License apache-2.0. Apache-2.0 (repository LICENSE).",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "sky-t1-32b-preview",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9385,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'NovaSky-AI/Sky-T1-32B-Preview', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "sky-t1-32b-preview",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9099,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'NovaSky-AI/Sky-T1-32B-Preview', instruction-tuned (📝), default setting; EuroEval version 15.8.2.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "reka-flash-3",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8994,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'RekaAI/reka-flash-3', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "reka-flash-3",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6886,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'RekaAI/reka-flash-3', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 15.9.2.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "reka-flash-3-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.874,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'RekaAI/reka-flash-3.1', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 15.12.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "reka-flash-3-1",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6909,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'RekaAI/reka-flash-3.1', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 15.12.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "afm-4-5b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8646,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'arcee-ai/AFM-4.5B', instruction-tuned (📝), default setting; EuroEval version 16.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "afm-4-5b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6024,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'arcee-ai/AFM-4.5B', instruction-tuned (📝), default setting; EuroEval version 16.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "trinity-mini",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9143,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'arcee-ai/Trinity-Mini', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 16.14.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "trinity-mini",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6874,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'arcee-ai/Trinity-Mini', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 16.14.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "mistral-nemo-minitron-8b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8312,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'nvidia/mistral-nemo-minitron-8b-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "stablelm-2-1-6b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.5061,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-09",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-2-1_6b-chat' (source=hub, precision=float16, model_sha b362325b3d08b5f37acc4a504b441b04414df818); published average 50.61 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-2-zephyr-1-6b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5582,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-2-zephyr-1_6b' (source=hub, precision=float16, model_sha c89d7d19e9781974793a7e9b0fe55bcabcf8abc5); published average 55.82 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-2-12b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.6809,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-09",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-2-12b-chat' (source=hub, precision=bfloat16, model_sha 2e874a70ad37d4b31bcac48fa2e24ce033e3fac7); published average 68.09 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-zephyr-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5437,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-zephyr-3b' (source=hub, precision=bfloat16, model_sha 015f44c1cdff647dbbd9e572661d05cef9da8ce1); published average 54.37 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "internlm2-5-1-8b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.4433,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-08-09",
    "note": "Open PT-LLM Leaderboard row 'internlm/internlm2_5-1_8b-chat' (source=hub, precision=bfloat16, model_sha 4426f00b854561fa60d555d2b628064b56bcb758); published average 44.33 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "nous-hermes-2-solar-10-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7163,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Nous-Hermes-2-SOLAR-10.7B' (source=hub, precision=bfloat16, model_sha 14c1fbe2f71acdcd58247b30d5439bd572d52386); published average 71.63 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "nous-hermes-2-yi-34b",
    "bench": "open-pt-llm-avg9",
    "score": 0.7242,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Nous-Hermes-2-Yi-34B' (source=hub, precision=bfloat16, model_sha fcb0a8847e76aea14aba9aa44009d4418ad7c18f); published average 72.42 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "notus-7b-v1",
    "bench": "open-pt-llm-avg9",
    "score": 0.6482,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'argilla/notus-7b-v1' (source=hub, precision=bfloat16, model_sha 30172203a2d41cb487bf7e2b92a821080783b2c9); published average 64.82 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "solar-pro-preview",
    "bench": "open-pt-llm-avg9",
    "score": 0.7369,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-09-18",
    "note": "Open PT-LLM Leaderboard row 'upstage/solar-pro-preview-instruct' (source=hub, precision=bfloat16, model_sha dd4bcf7006df9b1ce3f87711e702e4063832aae3); published average 73.69 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "nous-hermes-2-mistral-7b-dpo",
    "bench": "open-pt-llm-avg9",
    "score": 0.6675,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-27",
    "note": "Open PT-LLM Leaderboard row 'NousResearch/Nous-Hermes-2-Mistral-7B-DPO' (source=hub, precision=bfloat16, model_sha b66d6a6b24c9d59639b12b5df43020059f662aaf); published average 66.75 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision has the same weight files as the current main.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "jan-v1-4b",
    "bench": "open-pt-llm-avg9",
    "score": 0.752,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'janhq/Jan-v1-4B' (source=hub, precision=bfloat16, model_sha 3a6fedb2f282925db523e1ca9fbbdbc6d86ef552); published average 75.2 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "command-r7b-arabic-02-2025",
    "bench": "aragen-v3-3c3h",
    "score": 0.2786,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2026-02-23",
    "note": "Inception AraGen-v3 leaderboard entry 'CohereForAI/c4ai-command-r7b-arabic-02-2025'; 3C3H Score 0.2786 (95% CI 0.239201-0.319064, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20260223_094614; RM_Thinking_Traces=False.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "fanar-1-9b-instruct",
    "bench": "aragen-v3-3c3h",
    "score": 0.1618,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'QCRI/Fanar-1-9B-Instruct'; 3C3H Score 0.1618 (95% CI 0.12473-0.200761, n=308); judge claude-sonnet-4-20250514; 308/324 entries answered; run 20250930_192918; RM_Thinking_Traces=None. 16 of 324 entries failed (success ratio 0.95).",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "mmlu-pro",
    "score": 0.776,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMLU Pro', column 'DiffusionGemma 26B A4B' = 77.6%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmlu-pro",
    "score": 0.826,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMLU Pro', column 'Gemma 4 26B A4B' = 82.6%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "aime-2026",
    "score": 0.691,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'AIME 2026 no tools', column 'DiffusionGemma 26B A4B' = 69.1%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "aime-2026",
    "score": 0.883,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'AIME 2026 no tools', column 'Gemma 4 26B A4B' = 88.3%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.691,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'LiveCodeBench v6', column 'DiffusionGemma 26B A4B' = 69.1%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.771,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'LiveCodeBench v6', column 'Gemma 4 26B A4B' = 77.1%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "gpqa-diamond",
    "score": 0.732,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'GPQA Diamond', column 'DiffusionGemma 26B A4B' = 73.2%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "gpqa-diamond",
    "score": 0.823,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'GPQA Diamond', column 'Gemma 4 26B A4B' = 82.3%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "tau2-bench",
    "score": 0.562,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'Tau2 (average over 3)', column 'DiffusionGemma 26B A4B' = 56.2%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "tau2-bench",
    "score": 0.682,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'Tau2 (average over 3)', column 'Gemma 4 26B A4B' = 68.2%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "hle",
    "score": 0.11,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'HLE no tools', column 'DiffusionGemma 26B A4B' = 11.0%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "hle",
    "score": 0.087,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'HLE no tools', column 'Gemma 4 26B A4B' = 8.7%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "mmmlu",
    "score": 0.815,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMMLU', column 'DiffusionGemma 26B A4B' = 81.5%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmmlu",
    "score": 0.863,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMMLU', column 'Gemma 4 26B A4B' = 86.3%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "mmmu-pro",
    "score": 0.543,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMMU Pro', column 'DiffusionGemma 26B A4B' = 54.3%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mmmu-pro",
    "score": 0.738,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MMMU Pro', column 'Gemma 4 26B A4B' = 73.8%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "omnidocbench-1-5",
    "score": 0.319,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'OmniDocBench 1.5 (average edit distance, lower is better)', column 'DiffusionGemma 26B A4B' = 0.319. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "omnidocbench-1-5",
    "score": 0.149,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'OmniDocBench 1.5 (average edit distance, lower is better)', column 'Gemma 4 26B A4B' = 0.149. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "math-vision",
    "score": 0.705,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MATH-Vision', column 'DiffusionGemma 26B A4B' = 70.5%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "math-vision",
    "score": 0.824,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MATH-Vision', column 'Gemma 4 26B A4B' = 82.4%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "medxpertqa-mm",
    "score": 0.49,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MedXPertQA MM', column 'DiffusionGemma 26B A4B' = 49.0%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "medxpertqa-mm",
    "score": 0.581,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MedXPertQA MM', column 'Gemma 4 26B A4B' = 58.1%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "diffusiongemma-26b-a4b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.32,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MRCR v2 8 needle 128k (average)', column 'DiffusionGemma 26B A4B' = 32.0%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "mrcr-v2-8needle-128k",
    "score": 0.441,
    "source": "https://huggingface.co/google/diffusiongemma-26B-A4B-it/blob/f7f5b7f5fa82ffc52addd066915886d497f5517b/README.md",
    "by": "lab",
    "date": "2026-07-15",
    "note": "Google DiffusionGemma model card (Apache-2.0, pinned f7f5b7f), Benchmark Results table, row 'MRCR v2 8 needle 128k (average)', column 'Gemma 4 26B A4B' = 44.1%; divided by 100. Instruction-tuned models with the recommended Entropy Bound sampler; thinking/tool settings as labelled in the row. Codeforces ELO, HLE with search and BigBench Extra Hard (full set, not the micro average id) not used.",
    "source_id": "card3-google-diffusiongemma-26b-a4b-it"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "bfcl-v4",
    "score": 0.651,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BFCL‑V4', column 'Spark‑X2.5‑4B' = 65.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "bfcl-v4",
    "score": 0.469,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BFCL‑V4', column 'Spark‑X2.5‑1.7B' = 46.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "bfcl-v4",
    "score": 0.374,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BFCL‑V4', column 'Gemma4‑12B' = 37.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bfcl-v4",
    "score": 0.369,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BFCL‑V4', column 'Gemma4‑E4B' = 36.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "bfcl-v4",
    "score": 0.302,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BFCL‑V4', column 'Gemma4‑E2B' = 30.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "tau2-bench",
    "score": 0.751,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ²‑bench', column 'Spark‑X2.5‑4B' = 75.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "tau2-bench",
    "score": 0.653,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ²‑bench', column 'Spark‑X2.5‑1.7B' = 65.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "tau3-bench",
    "score": 0.304,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Spark‑X2.5‑4B' = 30.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "tau3-bench",
    "score": 0.201,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Spark‑X2.5‑1.7B' = 20.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "tau3-bench",
    "score": 0.093,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Qwen3.5‑9B' = 9.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "tau3-bench",
    "score": 0.067,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Qwen3.5‑4B' = 6.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "tau3-bench",
    "score": 0.041,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Qwen3.5‑2B' = 4.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "tau3-bench",
    "score": 0.133,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Gemma4‑12B' = 13.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau3-bench",
    "score": 0.101,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Gemma4‑E4B' = 10.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau3-bench",
    "score": 0.088,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'τ³‑bench', column 'Gemma4‑E2B' = 8.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "mcp-atlas",
    "score": 0.546,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Atlas', column 'Spark‑X2.5‑4B' = 54.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "mcp-atlas",
    "score": 0.234,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Atlas', column 'Spark‑X2.5‑1.7B' = 23.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "mcp-atlas",
    "score": 0.148,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Atlas', column 'Qwen3.5‑2B' = 14.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "mcp-atlas",
    "score": 0.126,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Atlas', column 'Gemma4‑E2B' = 12.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "mcpmark",
    "score": 0.142,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Mark', column 'Spark‑X2.5‑4B' = 14.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "mcpmark",
    "score": 0.023,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Mark', column 'Spark‑X2.5‑1.7B' = 2.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mcpmark",
    "score": 0.134,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Mark', column 'Qwen3.5‑9B' = 13.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mcpmark",
    "score": 0.125,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'MCP‑Mark', column 'Qwen3.5‑4B' = 12.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "browsecomp",
    "score": 0.409,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Spark‑X2.5‑4B' = 40.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "browsecomp",
    "score": 0.297,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Spark‑X2.5‑1.7B' = 29.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "browsecomp",
    "score": 0.083,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Qwen3.5‑9B' = 8.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "browsecomp",
    "score": 0.143,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Qwen3.5‑4B' = 14.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "browsecomp",
    "score": 0.031,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Qwen3.5‑2B' = 3.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "browsecomp",
    "score": 0.1,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Gemma4‑12B' = 10.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "browsecomp",
    "score": 0.083,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Gemma4‑E4B' = 8.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "browsecomp",
    "score": 0.037,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Agent', row 'BrowseComp', column 'Gemma4‑E2B' = 3.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.444,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Pro', column 'Spark‑X2.5‑4B' = 44.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.104,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Pro', column 'Spark‑X2.5‑1.7B' = 10.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.019,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Pro', column 'Qwen3.5‑2B' = 1.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "swe-bench-verified",
    "score": 0.416,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Verified', column 'Spark‑X2.5‑4B' = 41.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "swe-bench-verified",
    "score": 0.283,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Verified', column 'Spark‑X2.5‑1.7B' = 28.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "swe-bench-verified",
    "score": 0.068,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Verified', column 'Qwen3.5‑2B' = 6.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "swe-bench-multilingual",
    "score": 0.533,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Multilingual', column 'Spark‑X2.5‑4B' = 53.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "swe-bench-multilingual",
    "score": 0.233,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Multilingual', column 'Spark‑X2.5‑1.7B' = 23.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "swe-bench-multilingual",
    "score": 0.433,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Multilingual', column 'Qwen3.5‑9B' = 43.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "swe-bench-multilingual",
    "score": 0.277,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Multilingual', column 'Qwen3.5‑4B' = 27.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "swe-bench-multilingual",
    "score": 0.05,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SWE‑Bench Multilingual', column 'Qwen3.5‑2B' = 5.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "scicode-unspecified",
    "score": 0.347,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Spark‑X2.5‑4B' = 34.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "scicode-unspecified",
    "score": 0.182,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Spark‑X2.5‑1.7B' = 18.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "scicode-unspecified",
    "score": 0.24,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Qwen3.5‑4B' = 24.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "scicode-unspecified",
    "score": 0.06,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Qwen3.5‑2B' = 6.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "scicode-unspecified",
    "score": 0.398,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Gemma4‑12B' = 39.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "scicode-unspecified",
    "score": 0.275,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Gemma4‑E4B' = 27.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "scicode-unspecified",
    "score": 0.205,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Code', row 'SciCode', column 'Gemma4‑E2B' = 20.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "aime-2026",
    "score": 0.907,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'AIME 2026', column 'Spark‑X2.5‑4B' = 90.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "aime-2026",
    "score": 0.694,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'AIME 2026', column 'Spark‑X2.5‑1.7B' = 69.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "aime-2026",
    "score": 0.882,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'AIME 2026', column 'Qwen3.5‑9B' = 88.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "aime-2026",
    "score": 0.83,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'AIME 2026', column 'Qwen3.5‑4B' = 83.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "aime-2026",
    "score": 0.308,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'AIME 2026', column 'Qwen3.5‑2B' = 30.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "hmmt-feb-2026",
    "score": 0.812,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Spark‑X2.5‑4B' = 81.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "hmmt-feb-2026",
    "score": 0.484,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Spark‑X2.5‑1.7B' = 48.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "hmmt-feb-2026",
    "score": 0.708,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Qwen3.5‑9B' = 70.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hmmt-feb-2026",
    "score": 0.697,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Qwen3.5‑4B' = 69.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hmmt-feb-2026",
    "score": 0.215,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Qwen3.5‑2B' = 21.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "hmmt-feb-2026",
    "score": 0.656,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Gemma4‑12B' = 65.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "hmmt-feb-2026",
    "score": 0.342,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Gemma4‑E4B' = 34.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "hmmt-feb-2026",
    "score": 0.205,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'HMMT Feb 2026', column 'Gemma4‑E2B' = 20.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "imo-answerbench",
    "score": 0.742,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Spark‑X2.5‑4B' = 74.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "imo-answerbench",
    "score": 0.454,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Spark‑X2.5‑1.7B' = 45.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "imo-answerbench",
    "score": 0.698,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Qwen3.5‑9B' = 69.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "imo-answerbench",
    "score": 0.685,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Qwen3.5‑4B' = 68.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "imo-answerbench",
    "score": 0.572,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Gemma4‑12B' = 57.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "imo-answerbench",
    "score": 0.269,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Gemma4‑E4B' = 26.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "imo-answerbench",
    "score": 0.226,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'Math', row 'IMO‑AnswerBench', column 'Gemma4‑E2B' = 22.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "ifeval-unspecified",
    "score": 0.93,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFEval', column 'Spark‑X2.5‑4B' = 93.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "ifeval-unspecified",
    "score": 0.895,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFEval', column 'Spark‑X2.5‑1.7B' = 89.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "ifeval-unspecified",
    "score": 0.948,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFEval', column 'Gemma4‑12B' = 94.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifeval-unspecified",
    "score": 0.453,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFEval', column 'Gemma4‑E4B' = 45.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifeval-unspecified",
    "score": 0.348,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFEval', column 'Gemma4‑E2B' = 34.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "ifbench-unspecified",
    "score": 0.75,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFBench', column 'Spark‑X2.5‑4B' = 75.0; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "ifbench-unspecified",
    "score": 0.663,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFBench', column 'Spark‑X2.5‑1.7B' = 66.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifbench-unspecified",
    "score": 0.645,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFBench', column 'Qwen3.5‑9B' = 64.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifbench-unspecified",
    "score": 0.592,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFBench', column 'Qwen3.5‑4B' = 59.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifbench-unspecified",
    "score": 0.227,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'IFBench', column 'Gemma4‑E2B' = 22.7; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "hle",
    "score": 0.123,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Spark‑X2.5‑4B' = 12.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "hle",
    "score": 0.063,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Spark‑X2.5‑1.7B' = 6.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "hle",
    "score": 0.143,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Qwen3.5‑9B' = 14.3; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "hle",
    "score": 0.086,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Qwen3.5‑4B' = 8.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "hle",
    "score": 0.021,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Qwen3.5‑2B' = 2.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "hle",
    "score": 0.131,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Gemma4‑12B' = 13.1; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "hle",
    "score": 0.039,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Gemma4‑E4B' = 3.9; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "hle",
    "score": 0.025,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'HLE', column 'Gemma4‑E2B' = 2.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-4b",
    "bench": "gpqa-unspecified",
    "score": 0.674,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Spark‑X2.5‑4B' = 67.4; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "spark-x2-5-1-7b",
    "bench": "gpqa-unspecified",
    "score": 0.438,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "lab",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Spark‑X2.5‑1.7B' = 43.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-unspecified",
    "score": 0.772,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Qwen3.5‑9B' = 77.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gpqa-unspecified",
    "score": 0.672,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Qwen3.5‑4B' = 67.2; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "gpqa-unspecified",
    "score": 0.446,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Qwen3.5‑2B' = 44.6; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Qwen3.5 hybrid models in thinking mode. Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-12b",
    "bench": "gpqa-unspecified",
    "score": 0.728,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Gemma4‑12B' = 72.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "gpqa-unspecified",
    "score": 0.545,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Gemma4‑E4B' = 54.5; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "gpqa-unspecified",
    "score": 0.438,
    "source": "https://huggingface.co/XHToken/Spark-X2.5-4B/blob/0bcb35678590218655dff3765b9e61c83b35e9c4/README.md",
    "by": "independent",
    "date": "2026-09-15",
    "note": "Spark-X2.5-4B model card (Apache-2.0, pinned 0bcb356), benchmark table section 'General & Knowledge', row 'GPQA', column 'Gemma4‑E2B' = 43.8; divided by 100. Unstarred cell = the authors' own run ('* denotes reported results from publicly-released model cards / papers'); 'All evaluations are conducted in thinking mode.' Workspace Bench, VitaBench2.0, Gaokao 2026 and AA-LCR rows not used.",
    "source_id": "card3-xhtoken-spark-x2-5-4b"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "aime-2025",
    "score": 0.68,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2025, mean@32', column 'GigaChat 3.5 Ultra Instruct' = 68; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "aime-2025",
    "score": 0.89,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2025, mean@32', column 'GigaChat 3.5 Ultra Reasoning' = 89; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "aime-2025",
    "score": 0.8895,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2025, mean@32', column 'DeepSeek V4 Flash Preview Reasoning' = 88.95; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "aime-2026",
    "score": 0.67,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2026, mean@32', column 'GigaChat 3.5 Ultra Instruct' = 67; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "aime-2026",
    "score": 0.92,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2026, mean@32', column 'GigaChat 3.5 Ultra Reasoning' = 92; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "aime-2026",
    "score": 0.904,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'AIME 2026, mean@32', column 'DeepSeek V4 Flash Preview Reasoning' = 90.4; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.3667,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'HMMT 2025, mean@8', column 'GigaChat 3.5 Ultra Instruct' = 36.67; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.8313,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'HMMT 2025, mean@8', column 'GigaChat 3.5 Ultra Reasoning' = 83.13; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.9521,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'HMMT 2025, mean@8', column 'DeepSeek V4 Flash Preview Reasoning' = 95.21; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "imo-answerbench",
    "score": 0.32,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IMOAnswerBench\\*', column 'GigaChat 3.5 Ultra Instruct' = 32; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "imo-answerbench",
    "score": 0.73,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IMOAnswerBench\\*', column 'GigaChat 3.5 Ultra Reasoning' = 73; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "imo-answerbench",
    "score": 0.8575,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IMOAnswerBench\\*', column 'DeepSeek V4 Flash Preview Reasoning' = 85.75; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "gpqa-diamond",
    "score": 0.6111,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'GPQA-Diamond', column 'GigaChat 3.5 Ultra Instruct' = 61.11; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.8232,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'GPQA-Diamond', column 'GigaChat 3.5 Ultra Reasoning' = 82.32; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "gpqa-diamond",
    "score": 0.874,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'GPQA-Diamond', column 'DeepSeek V4 Flash Preview Reasoning' = 87.4; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "ifbench-unspecified",
    "score": 0.4366,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IFBench', column 'GigaChat 3.5 Ultra Instruct' = 43.66; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "ifbench-unspecified",
    "score": 0.77,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IFBench', column 'GigaChat 3.5 Ultra Reasoning' = 77; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "ifbench-unspecified",
    "score": 0.7333,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'IFBench', column 'DeepSeek V4 Flash Preview Reasoning' = 73.33; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "tau3-bench",
    "score": 0.5003,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'TAU3-bench\\*\\*', column 'GigaChat 3.5 Ultra Instruct' = 50.03; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "tau3-bench",
    "score": 0.478,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'TAU3-bench\\*\\*', column 'GigaChat 3.5 Ultra Reasoning' = 47.8; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "tau3-bench",
    "score": 0.677,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'TAU3-bench\\*\\*', column 'DeepSeek V4 Flash Preview Reasoning' = 67.7; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.562,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Live Code Bench v6', column 'GigaChat 3.5 Ultra Instruct' = 56.2; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.854,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Live Code Bench v6', column 'GigaChat 3.5 Ultra Reasoning' = 85.4; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.8787,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Live Code Bench v6', column 'DeepSeek V4 Flash Preview Reasoning' = 87.87; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "swe-bench-verified",
    "score": 0.426,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'SWE-bench Verified\\*\\*\\*\\*', column 'GigaChat 3.5 Ultra Instruct' = 42.6; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "swe-bench-verified",
    "score": 0.647,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'SWE-bench Verified\\*\\*\\*\\*', column 'GigaChat 3.5 Ultra Reasoning' = 64.7; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "swe-bench-verified",
    "score": 0.786,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'SWE-bench Verified\\*\\*\\*\\*', column 'DeepSeek V4 Flash Preview Reasoning' = 78.6; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-instruct",
    "bench": "terminal-bench-2",
    "score": 0.1348,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Terminal-Bench 2\\*\\*\\*\\*', column 'GigaChat 3.5 Ultra Instruct' = 13.48; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "gigachat-3-5-ultra-reasoning",
    "bench": "terminal-bench-2",
    "score": 0.303,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "lab",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Terminal-Bench 2\\*\\*\\*\\*', column 'GigaChat 3.5 Ultra Reasoning' = 30.3; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used.",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "terminal-bench-2",
    "score": 0.566,
    "source": "https://huggingface.co/ai-sage/GigaChat3.5-432B-A28B-Reasoning/blob/02f55379ac0b20abd9378791bffd6254725e6f5c/README.md",
    "by": "independent",
    "date": "2026-09-10",
    "note": "GigaChat 3.5 Reasoning model card (MIT, pinned 02f5537), Benchmark scores table, row 'Terminal-Bench 2\\*\\*\\*\\*', column 'DeepSeek V4 Flash Preview Reasoning' = 56.6; divided by 100. Card notes: IMOAnswerBench judged by Qwen-3-235B-Instruct-2507; TAU3 averaged over Airline, Retail, Telecom and Banking; SWE-bench Verified and Terminal-Bench 2 with mini-swe-agent and a three-hour timeout. StructEval, MERA-2.0, Function Calling V4, Natural Plan and Arena rows not used. DeepSeek column = Sber's own run of DeepSeek V4 Flash Preview in reasoning mode (see held_for_owner).",
    "source_id": "card3-ai-sage-gigachat3-5-432b-a28b-reasoning"
  },
  {
    "model": "ax-k2",
    "bench": "aime-2026",
    "score": 0.971,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'A.X K2' = 97.1; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "aime-2026",
    "score": 0.913,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'A.X K1' = 91.3; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "aime-2026",
    "score": 0.925,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'Qwen3.5-397B-A17B' = 92.5; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "aime-2026",
    "score": 0.854,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'Nemotron 3 Ultra' = 85.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "aime-2026",
    "score": 0.954,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'GLM-5.1' = 95.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "aime-2026",
    "score": 0.954,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'Kimi-K2.6' = 95.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "aime-2026",
    "score": 0.888,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'AIME26', column 'MiniMax-M2.7' = 88.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "livecodebench-v6",
    "score": 0.84,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'A.X K2' = 84.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "livecodebench-v6",
    "score": 0.749,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'A.X K1' = 74.9; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "livecodebench-v6",
    "score": 0.826,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'Qwen3.5-397B-A17B' = 82.6; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "livecodebench-v6",
    "score": 0.764,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'Nemotron 3 Ultra' = 76.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "livecodebench-v6",
    "score": 0.864,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'GLM-5.1' = 86.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "livecodebench-v6",
    "score": 0.865,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'Kimi-K2.6' = 86.5; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "livecodebench-v6",
    "score": 0.698,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'LiveCodeBench v6 (Feb–May)', column 'MiniMax-M2.7' = 69.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure). LiveCodeBench: 'we evaluate only the February–May 2025 subset'.",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "scicode-unspecified",
    "score": 0.41,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'A.X K2' = 41.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "scicode-unspecified",
    "score": 0.241,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'A.X K1' = 24.1; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "scicode-unspecified",
    "score": 0.42,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'Qwen3.5-397B-A17B' = 42.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "scicode-unspecified",
    "score": 0.399,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'Nemotron 3 Ultra' = 39.9; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "scicode-unspecified",
    "score": 0.438,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'GLM-5.1' = 43.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "scicode-unspecified",
    "score": 0.535,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'Kimi-K2.6' = 53.5; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "scicode-unspecified",
    "score": 0.47,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'SciCode', column 'MiniMax-M2.7' = 47.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "hle",
    "score": 0.278,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'A.X K2' = 27.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "hle",
    "score": 0.083,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'A.X K1' = 8.3; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "hle",
    "score": 0.273,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'Qwen3.5-397B-A17B' = 27.3; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "hle",
    "score": 0.266,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'Nemotron 3 Ultra' = 26.6; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "hle",
    "score": 0.28,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'GLM-5.1' = 28.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "hle",
    "score": 0.359,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'Kimi-K2.6' = 35.9; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "hle",
    "score": 0.281,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'Humanity’s Last Exam', column 'MiniMax-M2.7' = 28.1; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "gpqa-diamond",
    "score": 0.856,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'A.X K2' = 85.6; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "gpqa-diamond",
    "score": 0.764,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'A.X K1' = 76.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "gpqa-diamond",
    "score": 0.893,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'Qwen3.5-397B-A17B' = 89.3; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "gpqa-diamond",
    "score": 0.867,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'Nemotron 3 Ultra' = 86.7; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "gpqa-diamond",
    "score": 0.868,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'GLM-5.1' = 86.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "gpqa-diamond",
    "score": 0.911,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'Kimi-K2.6' = 91.1; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "gpqa-diamond",
    "score": 0.874,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'GPQA Diamond', column 'MiniMax-M2.7' = 87.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "ifbench-unspecified",
    "score": 0.759,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'A.X K2' = 75.9; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "ifbench-unspecified",
    "score": 0.632,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'A.X K1' = 63.2; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "ifbench-unspecified",
    "score": 0.788,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'Qwen3.5-397B-A17B' = 78.8; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "ifbench-unspecified",
    "score": 0.814,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'Nemotron 3 Ultra' = 81.4; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "ifbench-unspecified",
    "score": 0.763,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'GLM-5.1' = 76.3; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "ifbench-unspecified",
    "score": 0.76,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'Kimi-K2.6' = 76.0; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "ifbench-unspecified",
    "score": 0.757,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 6 'Comparison with strong open-weight LLMs', row 'IFBench', column 'MiniMax-M2.7' = 75.7; divided by 100. Sec. 5.1: 'We evaluate A.X K2—alongside its predecessor A.X K1 and leading open-weight baselines ... all models are evaluated in thinking mode' (row not starred, so not an Artificial Analysis figure).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k2",
    "bench": "global-mmlu-lite",
    "score": 0.8756,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'A.X K2', column 'Avg' = 87.56 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "ax-k1",
    "bench": "global-mmlu-lite",
    "score": 0.7775,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "lab",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'A.X K1', column 'Avg' = 77.75 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "global-mmlu-lite",
    "score": 0.9188,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'Qwen3.5-397B-A17B', column 'Avg' = 91.88 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "global-mmlu-lite",
    "score": 0.9094,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'Nemotron 3 Ultra', column 'Avg' = 90.94 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "glm-5-1",
    "bench": "global-mmlu-lite",
    "score": 0.9031,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'GLM-5.1', column 'Avg' = 90.31 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "kimi-k2-6",
    "bench": "global-mmlu-lite",
    "score": 0.9062,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'Kimi-K2.6', column 'Avg' = 90.62 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "minimax-m2-7",
    "bench": "global-mmlu-lite",
    "score": 0.8731,
    "source": "https://arxiv.org/html/2608.30181v1",
    "by": "independent",
    "date": "2026-08-31",
    "note": "A.X K2 Technical Report (arXiv 2608.30181v1, CC BY 4.0), Table 20 'Multilingual performance on Global-MMLU-Lite in thinking mode (exact match, %)', row 'MiniMax-M2.7', column 'Avg' = 87.31 (English/Spanish/Chinese/Japanese average); divided by 100. Authors' own runs (Sec. 5.1).",
    "source_id": "paper-2608.30181v1-ax-k2"
  },
  {
    "model": "gpt-5-4",
    "bench": "agentworldbench-overall",
    "score": 0.5825,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'GPT-5.4', column 'Overall' = 58.25; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "agentworldbench-overall",
    "score": 0.5659,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Claude Opus 4.8', column 'Overall' = 56.59; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "agentworldbench-overall",
    "score": 0.578,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Claude Opus 4.6', column 'Overall' = 57.80; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "agentworldbench-overall",
    "score": 0.5457,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Gemini 3.1 Pro', column 'Overall' = 54.57; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "agentworldbench-overall",
    "score": 0.5604,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Claude Sonnet 4.6', column 'Overall' = 56.04; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "glm-5-1",
    "bench": "agentworldbench-overall",
    "score": 0.5131,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'GLM-5.1', column 'Overall' = 51.31; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "kimi-k2-6",
    "bench": "agentworldbench-overall",
    "score": 0.5342,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Kimi K2.6', column 'Overall' = 53.42; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "minimax-m2-7",
    "bench": "agentworldbench-overall",
    "score": 0.4612,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'MiniMax-M2.7', column 'Overall' = 46.12; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "agentworldbench-overall",
    "score": 0.4773,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Qwen3.5-35B-A3B', column 'Overall' = 47.73; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "agentworldbench-overall",
    "score": 0.5474,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Qwen3.5-397B-A17B', column 'Overall' = 54.74; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "agentworldbench-overall",
    "score": 0.5081,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Qwen3.6-Plus', column 'Overall' = 50.81; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights).",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "mmlu-pro",
    "score": 0.824,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'MMLU-Pro', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 82.4. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "mmlu-pro",
    "score": 0.838,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'MMLU-Pro', column 'Nemotron-3-Super-120B-A12B-BF16' = 83.8. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "aime-2025",
    "score": 0.897,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'AIME25 (no tools)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 89.7. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "aime-2025",
    "score": 0.922,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'AIME25 (no tools)', column 'Nemotron-3-Super-120B-A12B-BF16' = 92.2. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "hmmt-feb-2025",
    "score": 0.934,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'HMMT Feb25 (no tools)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 93.4. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "hmmt-feb-2025",
    "score": 0.942,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'HMMT Feb25 (no tools)', column 'Nemotron-3-Super-120B-A12B-BF16' = 94.2. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "gpqa-unspecified",
    "score": 0.786,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'GPQA (no tools)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 78.6. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "gpqa-unspecified",
    "score": 0.805,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'GPQA (no tools)', column 'Nemotron-3-Super-120B-A12B-BF16' = 80.5. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "livecodebench-v5-202407-202412",
    "score": 0.811,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'LiveCodeBench (v5 2024-07↔2024-12)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 81.1. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "livecodebench-v5-202407-202412",
    "score": 0.821,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'LiveCodeBench (v5 2024-07↔2024-12)', column 'Nemotron-3-Super-120B-A12B-BF16' = 82.1. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "scicode-subtask",
    "score": 0.406,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'SciCode (subtask)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 40.6. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "scicode-subtask",
    "score": 0.423,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'SciCode (subtask)', column 'Nemotron-3-Super-120B-A12B-BF16' = 42.3. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "hle",
    "score": 0.165,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'HLE (no tools)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 16.5. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "hle",
    "score": 0.185,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'HLE (no tools)', column 'Nemotron-3-Super-120B-A12B-BF16' = 18.5. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.24,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'Terminal Bench (hard subset)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 24.0. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation. Nemotron-3-Super cell already recorded for this bench from its own card, so only the Puzzle cell is added.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "swe-bench-verified",
    "score": 0.569,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'SWE-Bench (OpenHands)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 56.9. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "swe-bench-verified",
    "score": 0.595,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'SWE-Bench (OpenHands)', column 'Nemotron-3-Super-120B-A12B-BF16' = 59.5. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "tau2-airline",
    "score": 0.558,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'TauBench V2 Airline', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 55.8. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "tau2-airline",
    "score": 0.569,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'TauBench V2 Airline', column 'Nemotron-3-Super-120B-A12B-BF16' = 56.9. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "tau2-retail",
    "score": 0.632,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'TauBench V2 Retail', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 63.2. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "tau2-retail",
    "score": 0.643,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'TauBench V2 Retail', column 'Nemotron-3-Super-120B-A12B-BF16' = 64.3. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "tau2-telecom",
    "score": 0.615,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'TauBench V2 Telecom', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 61.5. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation. Nemotron-3-Super cell already recorded for this bench from its own card, so only the Puzzle cell is added.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-labs-3-puzzle-75b-a9b",
    "bench": "ifbench-prompt",
    "score": 0.719,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'IFBench (prompt)', column 'Nemotron-Labs-3-Puzzle-75B-A9B-BF16' = 71.9. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "ifbench-prompt",
    "score": 0.734,
    "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16/blob/8fe5546888e9bd03fdbf52d808adebdfca901b52/README.md#benchmarks",
    "by": "lab",
    "date": "2026-06-24",
    "note": "NVIDIA Nemotron-Labs-3-Puzzle-75B-A9B model card (pinned 8fe5546), Benchmarks table, row 'IFBench (prompt)', column 'Nemotron-3-Super-120B-A12B-BF16' = 73.4. 'All evaluation results were collected via Nemo Evaluator SDK and for most benchmarks, the Nemo Skills Harness' (SWE-bench Verified with OpenHands; Terminal Bench hard subset 48 tasks). Vendor-reported (rule b; OpenMDW-1.1 card treated like the Poolside precedent), hand-recorded; date = repo creation.",
    "source_id": "first-party-nvidia-scores"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.372,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'LiveCodeBench v6', column 'Mellum2 Instruct' = 37.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.51,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'LiveCodeBench v6', column 'Qwen3.5 (4B)' = 51.0; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.637,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'LiveCodeBench v6', column 'Qwen3.5 (9B)' = 63.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.282,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'LiveCodeBench v6', column 'OLMo-3 (7B)' = 28.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.424,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'LiveCodeBench v6', column 'Ministral 3 (14B)' = 42.4; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "multipl-e",
    "score": 0.671,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MultiPL-E', column 'Mellum2 Instruct' = 67.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "multipl-e",
    "score": 0.51,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MultiPL-E', column 'Qwen3.5 (4B)' = 51.0; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "multipl-e",
    "score": 0.671,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MultiPL-E', column 'Qwen3.5 (9B)' = 67.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "multipl-e",
    "score": 0.361,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MultiPL-E', column 'OLMo-3 (7B)' = 36.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "multipl-e",
    "score": 0.715,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MultiPL-E', column 'Ministral 3 (14B)' = 71.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "bfcl-v4",
    "score": 0.442,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v4', column 'Mellum2 Instruct' = 44.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v4",
    "score": 0.52,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v4', column 'Qwen3.5 (4B)' = 52.0; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v4",
    "score": 0.606,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v4', column 'Qwen3.5 (9B)' = 60.6; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "bfcl-v4",
    "score": 0.198,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v4', column 'OLMo-3 (7B)' = 19.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "bfcl-v4",
    "score": 0.388,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v4', column 'Ministral 3 (14B)' = 38.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "bfcl-v3",
    "score": 0.663,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v3', column 'Mellum2 Instruct' = 66.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v3",
    "score": 0.641,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v3', column 'Qwen3.5 (4B)' = 64.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v3",
    "score": 0.705,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v3', column 'Qwen3.5 (9B)' = 70.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "bfcl-v3",
    "score": 0.419,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v3', column 'OLMo-3 (7B)' = 41.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "bfcl-v3",
    "score": 0.527,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'BFCL v3', column 'Ministral 3 (14B)' = 52.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "gsm-plus",
    "score": 0.805,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GSM-Plus', column 'Mellum2 Instruct' = 80.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gsm-plus",
    "score": 0.852,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GSM-Plus', column 'Qwen3.5 (4B)' = 85.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gsm-plus",
    "score": 0.879,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GSM-Plus', column 'Qwen3.5 (9B)' = 87.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "gsm-plus",
    "score": 0.858,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GSM-Plus', column 'OLMo-3 (7B)' = 85.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "gsm-plus",
    "score": 0.866,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GSM-Plus', column 'Ministral 3 (14B)' = 86.6; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "mmlu-redux",
    "score": 0.781,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MMLU-Redux', column 'Mellum2 Instruct' = 78.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-redux",
    "score": 0.875,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MMLU-Redux', column 'Qwen3.5 (4B)' = 87.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-redux",
    "score": 0.911,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MMLU-Redux', column 'Qwen3.5 (9B)' = 91.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "mmlu-redux",
    "score": 0.718,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MMLU-Redux', column 'OLMo-3 (7B)' = 71.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "mmlu-redux",
    "score": 0.859,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'MMLU-Redux', column 'Ministral 3 (14B)' = 85.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "gpqa-diamond",
    "score": 0.409,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GPQA Diamond', column 'Mellum2 Instruct' = 40.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gpqa-diamond",
    "score": 0.768,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GPQA Diamond', column 'Qwen3.5 (4B)' = 76.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.798,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GPQA Diamond', column 'Qwen3.5 (9B)' = 79.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "gpqa-diamond",
    "score": 0.409,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GPQA Diamond', column 'OLMo-3 (7B)' = 40.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "gpqa-diamond",
    "score": 0.586,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'GPQA Diamond', column 'Ministral 3 (14B)' = 58.6; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.758,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'IFEval', column 'Mellum2 Instruct' = 75.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifeval-unspecified",
    "score": 0.821,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'IFEval', column 'Qwen3.5 (4B)' = 82.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifeval-unspecified",
    "score": 0.839,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'IFEval', column 'Qwen3.5 (9B)' = 83.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "olmo-3-7b-instruct",
    "bench": "ifeval-unspecified",
    "score": 0.832,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'IFEval', column 'OLMo-3 (7B)' = 83.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "ministral-3-14b",
    "bench": "ifeval-unspecified",
    "score": 0.673,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Instruct/blob/273d9c1c6882ac8e1562e5b7e043857d35a787b6/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Instruct model card (Apache-2.0, pinned 273d9c1), post-training evaluation table (instruct, no-thinking variants), row 'IFEval', column 'Ministral 3 (14B)' = 67.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-instruct"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.699,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'LiveCodeBench v6', column 'Mellum2 Thinking' = 69.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.594,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'LiveCodeBench v6', column 'Qwen3.5 (4B)' = 59.4; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.683,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'LiveCodeBench v6', column 'Qwen3.5 (9B)' = 68.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.598,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'LiveCodeBench v6', column 'OLMo-3 (7B)' = 59.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.427,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'LiveCodeBench v6', column 'Ministral 3 (14B)' = 42.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "bfcl-v4",
    "score": 0.456,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v4', column 'Mellum2 Thinking' = 45.6; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v4",
    "score": 0.429,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v4', column 'Qwen3.5 (4B)' = 42.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v4",
    "score": 0.427,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v4', column 'Qwen3.5 (9B)' = 42.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "bfcl-v4",
    "score": 0.359,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v4', column 'Ministral 3 (14B)' = 35.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "bfcl-v3",
    "score": 0.694,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v3', column 'Mellum2 Thinking' = 69.4; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v3",
    "score": 0.739,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v3', column 'Qwen3.5 (4B)' = 73.9; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v3",
    "score": 0.685,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v3', column 'Qwen3.5 (9B)' = 68.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "bfcl-v3",
    "score": 0.522,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'BFCL v3', column 'Ministral 3 (14B)' = 52.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "gsm-plus",
    "score": 0.87,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GSM-Plus', column 'Mellum2 Thinking' = 87.0; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gsm-plus",
    "score": 0.893,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GSM-Plus', column 'Qwen3.5 (4B)' = 89.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gsm-plus",
    "score": 0.907,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GSM-Plus', column 'Qwen3.5 (9B)' = 90.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "gsm-plus",
    "score": 0.881,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GSM-Plus', column 'OLMo-3 (7B)' = 88.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "gsm-plus",
    "score": 0.865,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GSM-Plus', column 'Ministral 3 (14B)' = 86.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "mmlu-redux",
    "score": 0.862,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'MMLU-Redux', column 'Mellum2 Thinking' = 86.2; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "mmlu-redux",
    "score": 0.883,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'MMLU-Redux', column 'Qwen3.5 (4B)' = 88.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-redux",
    "score": 0.917,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'MMLU-Redux', column 'Qwen3.5 (9B)' = 91.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "mmlu-redux",
    "score": 0.713,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'MMLU-Redux', column 'OLMo-3 (7B)' = 71.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "mmlu-redux",
    "score": 0.844,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'MMLU-Redux', column 'Ministral 3 (14B)' = 84.4; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.576,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GPQA Diamond', column 'Mellum2 Thinking' = 57.6; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "gpqa-diamond",
    "score": 0.768,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GPQA Diamond', column 'Qwen3.5 (4B)' = 76.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.813,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GPQA Diamond', column 'Qwen3.5 (9B)' = 81.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "gpqa-diamond",
    "score": 0.293,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GPQA Diamond', column 'OLMo-3 (7B)' = 29.3; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "gpqa-diamond",
    "score": 0.46,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'GPQA Diamond', column 'Ministral 3 (14B)' = 46.0; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.765,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "lab",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'IFEval', column 'Mellum2 Thinking' = 76.5; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifeval-unspecified",
    "score": 0.871,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'IFEval', column 'Qwen3.5 (4B)' = 87.1; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifeval-unspecified",
    "score": 0.898,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'IFEval', column 'Qwen3.5 (9B)' = 89.8; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "olmo-3-7b-think",
    "bench": "ifeval-unspecified",
    "score": 0.847,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'IFEval', column 'OLMo-3 (7B)' = 84.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "ifeval-unspecified",
    "score": 0.597,
    "source": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/a7311550557e93cc706ab5dd3d879c1a11703ab4/README.md",
    "by": "independent",
    "date": "2026-08-13",
    "note": "JetBrains Mellum2-12B-A2.5B-Thinking model card (Apache-2.0, pinned a731155), post-training evaluation table (thinking variants), row 'IFEval', column 'Ministral 3 (14B)' = 59.7; divided by 100. 'All values self-reported by JetBrains.' Card notes: BFCL v4 is the macro-average of v1, v2, v3, web search and memory. AIME (mean of 2025 and 2026), EvalPlus mean, JetBrains pairwise, MixEval, BS-Bench and safety rows not used; Seed-Coder column not catalogued; SFT checkpoint column not used.",
    "source_id": "card3-jetbrains-mellum2-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.82,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'LiveCodeBench v6', column 'Mellum2.1 Thinking' = 82.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.694,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'LiveCodeBench v6', column 'Mellum2 Thinking' = 69.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.694,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'LiveCodeBench v6', column 'Gemma 4 (E4B)' = 69.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.754,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'LiveCodeBench v6', column 'Qwen3.5 (9B)' = 75.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "humaneval-plus",
    "score": 0.915,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'HumanEval+', column 'Mellum2.1 Thinking' = 91.5; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "humaneval-plus",
    "score": 0.909,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'HumanEval+', column 'Mellum2 Thinking' = 90.9; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "humaneval-plus",
    "score": 0.891,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'HumanEval+', column 'Gemma 4 (E4B)' = 89.1; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "humaneval-plus",
    "score": 0.896,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'HumanEval+', column 'Qwen3.5 (9B)' = 89.6; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "mbpp-plus",
    "score": 0.794,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MBPP+', column 'Mellum2.1 Thinking' = 79.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "mbpp-plus",
    "score": 0.754,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MBPP+', column 'Mellum2 Thinking' = 75.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mbpp-plus",
    "score": 0.709,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MBPP+', column 'Gemma 4 (E4B)' = 70.9; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mbpp-plus",
    "score": 0.698,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MBPP+', column 'Qwen3.5 (9B)' = 69.8; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "gsm-plus",
    "score": 0.883,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GSM-Plus', column 'Mellum2.1 Thinking' = 88.3; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "gsm-plus",
    "score": 0.871,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GSM-Plus', column 'Mellum2 Thinking' = 87.1; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "gsm-plus",
    "score": 0.874,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GSM-Plus', column 'Gemma 4 (E4B)' = 87.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gsm-plus",
    "score": 0.914,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GSM-Plus', column 'Qwen3.5 (9B)' = 91.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "swe-bench-verified",
    "score": 0.47,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Verified', column 'Mellum2.1 Thinking' = 47.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "swe-bench-verified",
    "score": 0.02,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Verified', column 'Mellum2 Thinking' = 2.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "swe-bench-verified",
    "score": 0.23,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Verified', column 'Gemma 4 (E4B)' = 23.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "swe-bench-verified",
    "score": 0.5,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Verified', column 'Qwen3.5 (9B)' = 50.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "terminal-bench-2.1",
    "score": 0.174,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'Terminal-Bench 2.1', column 'Mellum2.1 Thinking' = 17.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "terminal-bench-2.1",
    "score": 0.006,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'Terminal-Bench 2.1', column 'Mellum2 Thinking' = 0.6; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "terminal-bench-2.1",
    "score": 0.034,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'Terminal-Bench 2.1', column 'Gemma 4 (E4B)' = 3.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "terminal-bench-2.1",
    "score": 0.217,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'Terminal-Bench 2.1', column 'Qwen3.5 (9B)' = 21.7; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.28,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Pro', column 'Mellum2.1 Thinking' = 28.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Pro', column 'Mellum2 Thinking' = 0.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.04,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Pro', column 'Gemma 4 (E4B)' = 4.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.38,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'SWE-bench Pro', column 'Qwen3.5 (9B)' = 38.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "bfcl-v4",
    "score": 0.623,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'BFCL v4', column 'Mellum2.1 Thinking' = 62.3; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "bfcl-v4",
    "score": 0.496,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'BFCL v4', column 'Mellum2 Thinking' = 49.6; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bfcl-v4",
    "score": 0.525,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'BFCL v4', column 'Gemma 4 (E4B)' = 52.5; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v4",
    "score": 0.585,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'BFCL v4', column 'Qwen3.5 (9B)' = 58.5; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.906,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'IFEval', column 'Mellum2.1 Thinking' = 90.6; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "ifeval-unspecified",
    "score": 0.795,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'IFEval', column 'Mellum2 Thinking' = 79.5; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifeval-unspecified",
    "score": 0.908,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'IFEval', column 'Gemma 4 (E4B)' = 90.8; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifeval-unspecified",
    "score": 0.924,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'IFEval', column 'Qwen3.5 (9B)' = 92.4; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.646,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GPQA Diamond', column 'Mellum2.1 Thinking' = 64.6; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "gpqa-diamond",
    "score": 0.51,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GPQA Diamond', column 'Mellum2 Thinking' = 51.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "gpqa-diamond",
    "score": 0.531,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GPQA Diamond', column 'Gemma 4 (E4B)' = 53.1; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "gpqa-diamond",
    "score": 0.778,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'GPQA Diamond', column 'Qwen3.5 (9B)' = 77.8; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-1-12b-a2-5b-thinking",
    "bench": "mmlu-redux",
    "score": 0.878,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MMLU-Redux', column 'Mellum2.1 Thinking' = 87.8; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "mellum2-12b-a2-5b-thinking",
    "bench": "mmlu-redux",
    "score": 0.86,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "lab",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MMLU-Redux', column 'Mellum2 Thinking' = 86.0; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "mmlu-redux",
    "score": 0.849,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MMLU-Redux', column 'Gemma 4 (E4B)' = 84.9; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "mmlu-redux",
    "score": 0.895,
    "source": "https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking/blob/92ddae9fc7665e9f801d141d2e5a6b2caf2460c4/README.md",
    "by": "independent",
    "date": "2026-10-07",
    "note": "JetBrains Mellum2.1-12B-A2.5B-Thinking model card (Apache-2.0, pinned 92ddae9), evaluation table, row 'MMLU-Redux', column 'Qwen3.5 (9B)' = 89.5; divided by 100. 'All models were evaluated by JetBrains with the same pipeline in thinking mode.' AIME 25/26 (combined), WorkBench, ToolHop, MixEval-Hard and safety rows not used.",
    "source_id": "card3-jetbrains-mellum2-1-12b-a2-5b-thinking"
  },
  {
    "model": "translategemma-27b",
    "bench": "wmt24pp",
    "score": 0.844,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 27B, system 'TranslateGemma', column 'C22' = 84.4; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "gemma-3-27b",
    "bench": "wmt24pp",
    "score": 0.831,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 27B, system 'Gemma 3', column 'C22' = 83.1; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "translategemma-12b",
    "bench": "wmt24pp",
    "score": 0.835,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 12B, system 'TranslateGemma', column 'C22' = 83.5; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "gemma-3-12b",
    "bench": "wmt24pp",
    "score": 0.816,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 12B, system 'Gemma 3', column 'C22' = 81.6; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "translategemma-4b",
    "bench": "wmt24pp",
    "score": 0.801,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 4B, system 'TranslateGemma', column 'C22' = 80.1; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "gemma-3-4b",
    "bench": "wmt24pp",
    "score": 0.772,
    "source": "https://arxiv.org/pdf/2601.09012v2",
    "by": "lab",
    "date": "2026-01-15",
    "note": "TranslateGemma Technical Report (arXiv 2601.09012v2, CC BY 4.0), Table 1 'Automatic evaluation results using MetricX and Comet22 (C22) on WMT24++', size 4B, system 'Gemma 3', column 'C22' = 77.2; divided by 100. WMT24++ 55 language pairs; Comet22 (the wmt24pp id records the metric the source reports). Gemma 3 = the released instruction-tuned Gemma 3 checkpoints used as baselines.",
    "source_id": "paper-2601.09012v2-translategemma"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "humaneval",
    "score": 0.616,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-0.5B-Instruct', column HumanEval HE = 61.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "humaneval-plus",
    "score": 0.573,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-0.5B-Instruct', column HumanEval HE+ = 57.3. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "mbpp",
    "score": 0.524,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-0.5B-Instruct', column MBPP MBPP = 52.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "mbpp-plus",
    "score": 0.437,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-0.5B-Instruct', column MBPP MBPP+ = 43.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "bigcodebench-instruct",
    "score": 0.111,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-0.5B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 11.1. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "multipl-e",
    "score": 0.496,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-0.5B-Instruct', column 'Average' = 49.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "cruxeval-i",
    "score": 0.339,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-0.5B-Instruct', column 'Input-CoT' = 33.9. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "cruxeval-o",
    "score": 0.278,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-0.5B-Instruct', column 'Output-CoT' = 27.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "humaneval",
    "score": 0.707,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-1.5B-Instruct', column HumanEval HE = 70.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "humaneval-plus",
    "score": 0.665,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-1.5B-Instruct', column HumanEval HE+ = 66.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "mbpp",
    "score": 0.692,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-1.5B-Instruct', column MBPP MBPP = 69.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "mbpp-plus",
    "score": 0.594,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-1.5B-Instruct', column MBPP MBPP+ = 59.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "bigcodebench-instruct",
    "score": 0.325,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-1.5B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 32.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "multipl-e",
    "score": 0.567,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-1.5B-Instruct', column 'Average' = 56.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "cruxeval-i",
    "score": 0.454,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-1.5B-Instruct', column 'Input-CoT' = 45.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-1-5b",
    "bench": "cruxeval-o",
    "score": 0.375,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-1.5B-Instruct', column 'Output-CoT' = 37.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "humaneval",
    "score": 0.841,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-3B-Instruct', column HumanEval HE = 84.1. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "humaneval-plus",
    "score": 0.805,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-3B-Instruct', column HumanEval HE+ = 80.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "mbpp",
    "score": 0.736,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-3B-Instruct', column MBPP MBPP = 73.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "mbpp-plus",
    "score": 0.624,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-3B-Instruct', column MBPP MBPP+ = 62.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "bigcodebench-instruct",
    "score": 0.358,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-3B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 35.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "multipl-e",
    "score": 0.721,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-3B-Instruct', column 'Average' = 72.1. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "cruxeval-i",
    "score": 0.532,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-3B-Instruct', column 'Input-CoT' = 53.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "cruxeval-o",
    "score": 0.56,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-3B-Instruct', column 'Output-CoT' = 56.0. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "humaneval",
    "score": 0.835,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'CodeQwen1.5-7B-Chat', column HumanEval HE = 83.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "humaneval-plus",
    "score": 0.787,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'CodeQwen1.5-7B-Chat', column HumanEval HE+ = 78.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "mbpp",
    "score": 0.777,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'CodeQwen1.5-7B-Chat', column MBPP MBPP = 77.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "mbpp-plus",
    "score": 0.672,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'CodeQwen1.5-7B-Chat', column MBPP MBPP+ = 67.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "bigcodebench-instruct",
    "score": 0.396,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'CodeQwen1.5-7B-Chat', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 39.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "multipl-e",
    "score": 0.708,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'CodeQwen1.5-7B-Chat', column 'Average' = 70.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "cruxeval-i",
    "score": 0.44,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'CodeQwen1.5-7B-Chat', column 'Input-CoT' = 44.0. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "codeqwen1-5-7b-chat",
    "bench": "cruxeval-o",
    "score": 0.388,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'CodeQwen1.5-7B-Chat', column 'Output-CoT' = 38.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "humaneval",
    "score": 0.884,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-7B-Instruct', column HumanEval HE = 88.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "humaneval-plus",
    "score": 0.841,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-7B-Instruct', column HumanEval HE+ = 84.1. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "mbpp",
    "score": 0.835,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-7B-Instruct', column MBPP MBPP = 83.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "mbpp-plus",
    "score": 0.717,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-7B-Instruct', column MBPP MBPP+ = 71.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "bigcodebench-instruct",
    "score": 0.41,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-7B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 41.0. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "multipl-e",
    "score": 0.765,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-7B-Instruct', column 'Average' = 76.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "cruxeval-i",
    "score": 0.658,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-7B-Instruct', column 'Input-CoT' = 65.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-7b",
    "bench": "cruxeval-o",
    "score": 0.659,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-7B-Instruct', column 'Output-CoT' = 65.9. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "humaneval",
    "score": 0.896,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-14B-Instruct', column HumanEval HE = 89.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "humaneval-plus",
    "score": 0.872,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-14B-Instruct', column HumanEval HE+ = 87.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "mbpp",
    "score": 0.862,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-14B-Instruct', column MBPP MBPP = 86.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "mbpp-plus",
    "score": 0.728,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-14B-Instruct', column MBPP MBPP+ = 72.8. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "bigcodebench-instruct",
    "score": 0.484,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-14B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 48.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "multipl-e",
    "score": 0.796,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-14B-Instruct', column 'Average' = 79.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "cruxeval-i",
    "score": 0.695,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-14B-Instruct', column 'Input-CoT' = 69.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-14b",
    "bench": "cruxeval-o",
    "score": 0.795,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-14B-Instruct', column 'Output-CoT' = 79.5. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "humaneval",
    "score": 0.927,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-32B-Instruct', column HumanEval HE = 92.7. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "humaneval-plus",
    "score": 0.872,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-32B-Instruct', column HumanEval HE+ = 87.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "mbpp",
    "score": 0.902,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-32B-Instruct', column MBPP MBPP = 90.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "mbpp-plus",
    "score": 0.751,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-32B-Instruct', column MBPP MBPP+ = 75.1. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "bigcodebench-instruct",
    "score": 0.496,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 16 (instruct models, code generation), row 'Qwen2.5-Coder-32B-Instruct', column BigCodeBench Full (caption: 'For bigcodebench here, we report “instruct” tasks score') = 49.6. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "multipl-e",
    "score": 0.794,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 17 (instruct-format MultiPL-E; Python, Java, C++, C#, TS, JS, PHP, Bash), row 'Qwen2.5-Coder-32B-Instruct', column 'Average' = 79.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "cruxeval-i",
    "score": 0.752,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-32B-Instruct', column 'Input-CoT' = 75.2. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "cruxeval-o",
    "score": 0.834,
    "source": "https://arxiv.org/html/2409.12186v3",
    "by": "lab",
    "date": "2024-11-12",
    "note": "Qwen2.5-Coder Technical Report, arXiv 2409.12186v3 (arXiv non-exclusive licence -> rule b), HTML Table 18 (CRUXEval of instruct models), row 'Qwen2.5-Coder-32B-Instruct', column 'Output-CoT' = 83.4. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "mmmu",
    "score": 0.411,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMMU val', column 'Qwen2-VL-2B' = 41.1. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "docvqa",
    "score": 0.901,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'DocVQA test', column 'Qwen2-VL-2B' = 90.1. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "infographicvqa",
    "score": 0.655,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'InfoVQA test', column 'Qwen2-VL-2B' = 65.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "chartqa",
    "score": 0.735,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'ChartQA test', column 'Qwen2-VL-2B' = 73.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "textvqa",
    "score": 0.797,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'TextVQA val', column 'Qwen2-VL-2B' = 79.7. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "ocrbench",
    "score": 0.809,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'OCRBench' (0-1000 scale; value/10 stored as a percentage like the other first-party-qwen-scores OCRBench rows), column 'Qwen2-VL-2B' = 809. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "realworldqa",
    "score": 0.629,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'RealWorldQA', column 'Qwen2-VL-2B' = 62.9. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "mmstar",
    "score": 0.48,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMStar', column 'Qwen2-VL-2B' = 48.0. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "hallusionbench",
    "score": 0.417,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'HallBench avg', column 'Qwen2-VL-2B' = 41.7. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "mathvista-mini",
    "score": 0.43,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MathVista testmini', column 'Qwen2-VL-2B' = 43.0. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "math-vision",
    "score": 0.124,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MathVision', column 'Qwen2-VL-2B' = 12.4. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "mmmu-pro",
    "score": 0.376,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMMU-Pro', column 'Qwen2-VL-2B' = 37.6. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "mvbench",
    "score": 0.632,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'MVBench', column 'Qwen2-VL-2B' = 63.2. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "videomme-without-sub",
    "score": 0.556,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', without-subtitles value, column 'Qwen2-VL-2B' = 55.6. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "videomme-with-sub",
    "score": 0.604,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', with-subtitles value, column 'Qwen2-VL-2B' = 60.4. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mmmu",
    "score": 0.541,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMMU val', column 'Qwen2-VL-7B' = 54.1. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "docvqa",
    "score": 0.945,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'DocVQA test', column 'Qwen2-VL-7B' = 94.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "infographicvqa",
    "score": 0.765,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'InfoVQA test', column 'Qwen2-VL-7B' = 76.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "chartqa",
    "score": 0.83,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'ChartQA test', column 'Qwen2-VL-7B' = 83.0. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "textvqa",
    "score": 0.843,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'TextVQA val', column 'Qwen2-VL-7B' = 84.3. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "ocrbench",
    "score": 0.866,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'OCRBench' (0-1000 scale; value/10 stored as a percentage like the other first-party-qwen-scores OCRBench rows), column 'Qwen2-VL-7B' = 866. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "realworldqa",
    "score": 0.701,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'RealWorldQA', column 'Qwen2-VL-7B' = 70.1. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mmstar",
    "score": 0.607,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMStar', column 'Qwen2-VL-7B' = 60.7. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "hallusionbench",
    "score": 0.506,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'HallBench avg', column 'Qwen2-VL-7B' = 50.6. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mathvista-mini",
    "score": 0.582,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MathVista testmini', column 'Qwen2-VL-7B' = 58.2. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "math-vision",
    "score": 0.163,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MathVision', column 'Qwen2-VL-7B' = 16.3. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mmmu-pro",
    "score": 0.435,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MMMU-Pro', column 'Qwen2-VL-7B' = 43.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "mvbench",
    "score": 0.67,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'MVBench', column 'Qwen2-VL-7B' = 67.0. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "videomme-without-sub",
    "score": 0.633,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', without-subtitles value, column 'Qwen2-VL-7B' = 63.3. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-7b",
    "bench": "videomme-with-sub",
    "score": 0.69,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', with-subtitles value, column 'Qwen2-VL-7B' = 69.0. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "docvqa",
    "score": 0.965,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'DocVQA test', column 'Qwen2-VL-72B' = 96.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "infographicvqa",
    "score": 0.845,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'InfoVQA test', column 'Qwen2-VL-72B' = 84.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "chartqa",
    "score": 0.883,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'ChartQA test', column 'Qwen2-VL-72B' = 88.3. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "textvqa",
    "score": 0.855,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'TextVQA val', column 'Qwen2-VL-72B' = 85.5. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "ocrbench",
    "score": 0.877,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'OCRBench' (0-1000 scale; value/10 stored as a percentage like the other first-party-qwen-scores OCRBench rows), column 'Qwen2-VL-72B' = 877. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "math-vision",
    "score": 0.259,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 2 row 'MathVision', column 'Qwen2-VL-72B' = 25.9. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "mvbench",
    "score": 0.736,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'MVBench', column 'Qwen2-VL-72B' = 73.6. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "videomme-without-sub",
    "score": 0.712,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', without-subtitles value, column 'Qwen2-VL-72B' = 71.2. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-vl-72b",
    "bench": "videomme-with-sub",
    "score": 0.778,
    "source": "https://arxiv.org/html/2409.12191v2",
    "by": "lab",
    "date": "2024-10-03",
    "note": "Qwen2-VL report, arXiv 2409.12191v2 (non-exclusive licence -> rule b), HTML Table 4 row 'Video-MME (wo/w subs)', with-subtitles value, column 'Qwen2-VL-72B' = 77.8. Instruct checkpoints (the released Qwen2-VL-*-Instruct models). Pooled with this ledger entry's existing Qwen2.5-VL/Qwen3-VL report rows. 72B cells already recorded from the Qwen2.5-VL report (MMMU, MMMU-Pro, MathVista, MMStar, HallBench, RealWorldQA) are not duplicated. Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "gsm8k",
    "score": 0.641,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-1.5B-Instruct', column 'GSM8K' = 64.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "math-subset-unspecified",
    "score": 0.251,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-1.5B-Instruct', column 'MATH' = 25.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-1-5b",
    "bench": "minerva-math",
    "score": 0.055,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-1.5B-Instruct', column 'Minerva Math' = 5.5 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-7b",
    "bench": "gsm8k",
    "score": 0.857,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-7B-Instruct', column 'GSM8K' = 85.7 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-7b",
    "bench": "math-subset-unspecified",
    "score": 0.529,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-7B-Instruct', column 'MATH' = 52.9 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-7b",
    "bench": "minerva-math",
    "score": 0.195,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-7B-Instruct', column 'Minerva Math' = 19.5 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-72b",
    "bench": "gsm8k",
    "score": 0.932,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-72B-Instruct', column 'GSM8K' = 93.2 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-72b",
    "bench": "math-subset-unspecified",
    "score": 0.69,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-72B-Instruct', column 'MATH' = 69.0 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-72b",
    "bench": "minerva-math",
    "score": 0.316,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-72B-Instruct', column 'Minerva Math' = 31.6 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-1-5b",
    "bench": "gsm8k",
    "score": 0.842,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-1.5B-Instruct', column 'GSM8K' = 84.2 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-1-5b",
    "bench": "math-subset-unspecified",
    "score": 0.694,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-1.5B-Instruct', column 'MATH' = 69.4 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-1-5b",
    "bench": "minerva-math",
    "score": 0.294,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-1.5B-Instruct', column 'Minerva Math' = 29.4 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-7b",
    "bench": "gsm8k",
    "score": 0.899,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-7B-Instruct', column 'GSM8K' = 89.9 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-7b",
    "bench": "math-subset-unspecified",
    "score": 0.751,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-7B-Instruct', column 'MATH' = 75.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-7b",
    "bench": "minerva-math",
    "score": 0.346,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-7B-Instruct', column 'Minerva Math' = 34.6 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-72b",
    "bench": "gsm8k",
    "score": 0.967,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-72B-Instruct', column 'GSM8K' = 96.7 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-72b",
    "bench": "math-subset-unspecified",
    "score": 0.84,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-72B-Instruct', column 'MATH' = 84.0 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-math-72b",
    "bench": "minerva-math",
    "score": 0.401,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2-Math-72B-Instruct', column 'Minerva Math' = 40.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-1-5b",
    "bench": "gsm8k",
    "score": 0.848,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-1.5B-Instruct', column 'GSM8K' = 84.8 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-1-5b",
    "bench": "math-subset-unspecified",
    "score": 0.758,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-1.5B-Instruct', column 'MATH' = 75.8 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-1-5b",
    "bench": "minerva-math",
    "score": 0.294,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-1.5B-Instruct', column 'Minerva Math' = 29.4 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-7b",
    "bench": "gsm8k",
    "score": 0.952,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-7B-Instruct', column 'GSM8K' = 95.2 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-7b",
    "bench": "math-subset-unspecified",
    "score": 0.836,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-7B-Instruct', column 'MATH' = 83.6 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-7b",
    "bench": "minerva-math",
    "score": 0.371,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-7B-Instruct', column 'Minerva Math' = 37.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-72b",
    "bench": "gsm8k",
    "score": 0.959,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-72B-Instruct', column 'GSM8K' = 95.9 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-72b",
    "bench": "math-subset-unspecified",
    "score": 0.859,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-72B-Instruct', column 'MATH' = 85.9 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-math-72b",
    "bench": "minerva-math",
    "score": 0.441,
    "source": "https://arxiv.org/html/2409.12122v1",
    "by": "lab",
    "date": "2024-09-18",
    "note": "Qwen2.5-Math Technical Report, arXiv 2409.12122v1 (non-exclusive licence -> rule b), HTML Table 3 (instruct models, English benchmarks), section 'Chain-of-Thought' (caption: zero-shot pass@1 for these benchmarks), row 'Qwen2.5-Math-72B-Instruct', column 'Minerva Math' = 44.1 (the plain pass@1 line, not the maj@8 / rm@8 lines). Vendor-reported (rule b): hand-recorded from the Qwen team report for Qwen's own model; other makers' columns not used.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen2-5-coder-0-5b",
    "bench": "aider-code-editing",
    "score": 0.143,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-10-29",
    "note": "Published entry 'Qwen2.5-Coder-0.5B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 14.3 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=58bd375; command=aider --model openai/Qwen2.5-Coder-0.5B-Instruct; run=2024-10-29-00-29-09--Qwen2.5-Coder-0.5B-Instruct. OpenAI-compatible endpoint serving Qwen2.5-Coder-0.5B-Instruct (not a quantised Ollama/GGUF entry); run after the model card release. Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-5-coder-3b",
    "bench": "aider-code-editing",
    "score": 0.391,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-11-04",
    "note": "Published entry 'Qwen2.5-Coder-3B-Instruct'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 39.1 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.59.2.dev; evaluator commit=0ba3647; command=aider --model openai/Qwen2.5-Coder-3B-Instruct; run=2024-11-04-02-25-32--Qwen2.5-Coder-3B-Instruct. OpenAI-compatible endpoint serving Qwen2.5-Coder-3B-Instruct (not a quantised Ollama/GGUF entry). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "qwen2-vl-2b",
    "bench": "vectara-hhem-factual-consistency-2025",
    "score": 0.917,
    "source": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md",
    "by": "independent",
    "date": "2025-10-16",
    "note": "Vectara hallucination leaderboard, previous dataset (branch hhem-2.3-old-dataset @9ceb782, Apache-2.0, last updated 16 October 2025), row 'Qwen2-VL-2B-Instruct': factual consistency rate 91.7% (hallucination rate 8.3%), answer rate 100.0%, average summary length 81.8 words. HHEM-2.3 judge. README: 'Qwen/Qwen2-VL-2B-Instruct ... accessed via Hugging Face's checkpoints'.",
    "source_id": "vectara-hallucination-leaderboard-apache"
  },
  {
    "model": "qwen1-5-0-5b-chat",
    "bench": "livebench-20240726",
    "score": 0.0521,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-0.5b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 5.21. Category means: math 4.4, coding 0.0, reasoning 2.7, language 2.9, instruction_following 21.3, data_analysis 0.0. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-0-5b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.2538,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-22",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-0.5B-Chat' (source=hub, precision=bfloat16, model_sha 7f630fd18dccab574ab1b78411a8753f989a55ac); published average 25.38 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen1-5-0-5b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3744,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen1.5-0.5B-Chat', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "qwen1-5-1-8b-chat",
    "bench": "livebench-20240726",
    "score": 0.0604,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen1.5-1.8b-chat'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 6.04. Category means: math 3.5, coding 0.0, reasoning 3.3, language 3.2, instruction_following 22.9, data_analysis 3.3. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "qwen1-5-1-8b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.4322,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-22",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-1.8B-Chat' (source=hub, precision=bfloat16, model_sha c9a26c4b56507014e6c42a4d543de01c272ba165); published average 43.22 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen1-5-1-8b-chat",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5719,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'Qwen/Qwen1.5-1.8B-Chat', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "qwen1-5-1-8b-chat",
    "bench": "alpacaeval2-lc",
    "score": 0.0259,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'Qwen1.5-1.8B-Chat' (mode 'verified'), length_controlled_winrate = 2.59 (LC standard error not listed); stored /100. 805 AlpacaEval instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview (the reference, LC 50.00 by definition). Raw win rate 3.71; average length 2673.",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen1-5-moe-a2-7b-chat",
    "bench": "open-pt-llm-avg9",
    "score": 0.626,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-12",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen1.5-MoE-A2.7B-Chat' (source=hub, precision=bfloat16, model_sha 5044110f02850ea174a24c890d4b453d10783647); published average 62.6 (divided by 100). Local harness run. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen2-math-72b",
    "bench": "livebench-20240726",
    "score": 0.3424,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'qwen2-math-72b-instruct'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 34.24. Category means: math 39.1, coding 41.2, reasoning 29.3, language 20.3, instruction_following 51.5, data_analysis 24.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "openthinker-32b",
    "bench": "aime-2024",
    "score": 0.68,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table; same columns as the existing rows of this source (AMC23 unused; LCBv2 now has an id but the existing rows of this card were not extended to it here). Row 'OpenThinker-32B', column 'AIME24' = 68.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "aime-2025",
    "score": 0.493,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table; same columns as the existing rows of this source (AMC23 unused; LCBv2 now has an id but the existing rows of this card were not extended to it here). Row 'OpenThinker-32B', column 'AIME25' = 49.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "math-500",
    "score": 0.906,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table; same columns as the existing rows of this source (AMC23 unused; LCBv2 now has an id but the existing rows of this card were not extended to it here). Row 'OpenThinker-32B', column 'MATH500' = 90.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "gpqa-diamond",
    "score": 0.635,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-32B/blob/0aefafe3ee27e073e14fb94a0563795fddf38c50/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-32B model card (Apache-2.0, pinned 0aefafe), results table; same columns as the existing rows of this source (AMC23 unused; LCBv2 now has an id but the existing rows of this card were not extended to it here). Row 'OpenThinker-32B', column 'GPQA-D' = 63.5; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-32b"
  },
  {
    "model": "openthinker2-7b",
    "bench": "aime-2024",
    "score": 0.5,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker2-7B', column 'AIME24' = 50.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker2-7b",
    "bench": "aime-2025",
    "score": 0.333,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker2-7B', column 'AIME25' = 33.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker2-7b",
    "bench": "math-500",
    "score": 0.884,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker2-7B', column 'MATH500' = 88.4; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker2-7b",
    "bench": "gpqa-diamond",
    "score": 0.493,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker2-7B', column 'GPQA-D' = 49.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker2-7b",
    "bench": "livecodebench-v2",
    "score": 0.556,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker2-7B', column 'LCBv2' = 55.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "aime-2024",
    "score": 0.313,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker-7B', column 'AIME24' = 31.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "aime-2025",
    "score": 0.233,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker-7B', column 'AIME25' = 23.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "math-500",
    "score": 0.832,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker-7B', column 'MATH500' = 83.2; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "gpqa-diamond",
    "score": 0.429,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker-7B', column 'GPQA-D' = 42.9; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "livecodebench-v2",
    "score": 0.38,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'OpenThinker-7B', column 'LCBv2' = 38.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2024",
    "score": 0.573,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'AIME24' = 57.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2025",
    "score": 0.333,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'AIME25' = 33.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-500",
    "score": 0.896,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'MATH500' = 89.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "gpqa-diamond",
    "score": 0.473,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'GPQA-D' = 47.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "livecodebench-v2",
    "score": 0.484,
    "source": "https://huggingface.co/open-thoughts/OpenThinker2-7B/blob/57922a00cca8bae9776277bc51c3aceedacc643c/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker2-7B model card (Apache-2.0, pinned 57922a0), results table. OlympicCoder-7B and OpenR1-Qwen-7B rows not catalogued; AMC23 has no id. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'LCBv2' = 48.4; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker2-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "aime-2024",
    "score": 0.313,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'OpenThinker-7B', column 'AIME24' = 31.3; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "math-500",
    "score": 0.83,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'OpenThinker-7B', column 'MATH500' = 83.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "gpqa-diamond",
    "score": 0.424,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'OpenThinker-7B', column 'GPQA-Diamond' = 42.4; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "openthinker-7b",
    "bench": "livecodebench-v2",
    "score": 0.399,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'OpenThinker-7B', column 'LCBv2 All' = 39.9; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "aime-2024",
    "score": 0.6,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'AIME24' = 60; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-500",
    "score": 0.882,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'MATH500' = 88.2; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "gpqa-diamond",
    "score": 0.469,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'GPQA-Diamond' = 46.9; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "livecodebench-v2",
    "score": 0.501,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'DeepSeek-R1-Distill-Qwen-7B', column 'LCBv2 All' = 50.1; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "aime-2024",
    "score": 0.087,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'gpt-4o-0513', column 'AIME24' = 8.7; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "math-500",
    "score": 0.758,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'gpt-4o-0513', column 'MATH500' = 75.8; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "gpqa-diamond",
    "score": 0.465,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'gpt-4o-0513', column 'GPQA-Diamond' = 46.5; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "livecodebench-v2",
    "score": 0.505,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'gpt-4o-0513', column 'LCBv2 All' = 50.5; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "o1-mini",
    "bench": "aime-2024",
    "score": 0.64,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'o1-mini', column 'AIME24' = 64; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "o1-mini",
    "bench": "math-500",
    "score": 0.856,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'o1-mini', column 'MATH500' = 85.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "o1-mini",
    "bench": "gpqa-diamond",
    "score": 0.6,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'o1-mini', column 'GPQA-Diamond' = 60; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "o1-mini",
    "bench": "livecodebench-v2",
    "score": 0.728,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-7B/blob/4ba9dee9b7e6e4796c6bccf6e15c2f5b60843e1e/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-7B model card (Apache-2.0, pinned 4ba9dee), results table ('gpt-4o-0513' = gpt-4o-2024-05-13; o1-mini at its API default). Bespoke-Stratos-7B not catalogued; LCBv2 Easy/Medium/Hard subsets unused. Row 'o1-mini', column 'LCBv2 All' = 72.8; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-7b"
  },
  {
    "model": "s1-1-32b",
    "bench": "aime-2024",
    "score": 0.647,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 's1.1-32B', column 'AIME24 I/II' = 64.7; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "math-500",
    "score": 0.89,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 's1.1-32B', column 'MATH500' = 89.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "gpqa-diamond",
    "score": 0.601,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 's1.1-32B', column 'GPQA Diamond' = 60.1; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "s1-1-32b",
    "bench": "livecodebench-v2",
    "score": 0.655,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 's1.1-32B', column 'LCBv2' = 65.5; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "aime-2024",
    "score": 0.767,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'DeepSeek-R1-Distill-Qwen-32B', column 'AIME24 I/II' = 76.7; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "math-500",
    "score": 0.894,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'DeepSeek-R1-Distill-Qwen-32B', column 'MATH500' = 89.4; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "gpqa-diamond",
    "score": 0.576,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'DeepSeek-R1-Distill-Qwen-32B', column 'GPQA Diamond' = 57.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "livecodebench-v2",
    "score": 0.712,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "independent",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'DeepSeek-R1-Distill-Qwen-32B', column 'LCBv2' = 71.2; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "aime-2024",
    "score": 0.66,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'OpenThinker-32B', column 'AIME24 I/II' = 66.0; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "math-500",
    "score": 0.906,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'OpenThinker-32B', column 'MATH500' = 90.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "gpqa-diamond",
    "score": 0.616,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'OpenThinker-32B', column 'GPQA Diamond' = 61.6; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "openthinker-32b",
    "bench": "livecodebench-v2",
    "score": 0.689,
    "source": "https://huggingface.co/open-thoughts/OpenThinker-32B/blob/840749ca114a8f02be56f378d2999ad6d9e42d4f/README.md",
    "by": "lab",
    "date": "2025-06-05",
    "note": "OpenThinker-32B model card (Apache-2.0, pinned 840749c), results table. 'AIME25 I' column skipped (one of the two AIME 2025 papers only); LIMO-32B and s1-32B rows not catalogued. Row 'OpenThinker-32B', column 'LCBv2' = 68.9; divided by 100. Open Thoughts' own run: card statement: 'The numbers reported in the table below are evaluated with our open-source tool Evalchemy.' (covers every row, so comparator rows are the authors' own runs).",
    "source_id": "card3-open-thoughts-openthinker-32b"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "arc-challenge",
    "score": 0.6945,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'ARC-C' (0-shot), column 'Hermes 3 405B' = 69.45. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "arc-challenge",
    "score": 0.6553,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'ARC-C' (0-shot), column 'Hermes 3 70B' = 65.53. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "arc-challenge",
    "score": 0.5811,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'ARC-C' (0-shot), column 'Hermes 3 8B' = 58.11. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "boolq",
    "score": 0.8893,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BoolQ' (0-shot), column 'Hermes 3 405B' = 88.93. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "boolq",
    "score": 0.8804,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BoolQ' (0-shot), column 'Hermes 3 70B' = 88.04. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "boolq",
    "score": 0.8495,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BoolQ' (0-shot), column 'Hermes 3 8B' = 84.95. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "bbh",
    "score": 0.7537,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BBH' (3-shot), column 'Hermes 3 405B' = 75.37. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "bbh",
    "score": 0.6782,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BBH' (3-shot), column 'Hermes 3 70B' = 67.82. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "bbh",
    "score": 0.5294,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'BBH' (3-shot), column 'Hermes 3 8B' = 52.94. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "gpqa-unspecified",
    "score": 0.4484,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'GPQA' (0-shot), column 'Hermes 3 405B' = 44.84. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "gpqa-unspecified",
    "score": 0.3767,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'GPQA' (0-shot), column 'Hermes 3 70B' = 37.67. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "gpqa-unspecified",
    "score": 0.2936,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'GPQA' (0-shot), column 'Hermes 3 8B' = 29.36. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "hellaswag",
    "score": 0.9019,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Hellaswag' (10-shot), column 'Hermes 3 405B' = 90.19. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "hellaswag",
    "score": 0.8819,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Hellaswag' (10-shot), column 'Hermes 3 70B' = 88.19. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "hellaswag",
    "score": 0.8283,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Hellaswag' (10-shot), column 'Hermes 3 8B' = 82.83. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "ifeval-unspecified",
    "score": 0.8487,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'IFEval' (Strict), column 'Hermes 3 405B' = 84.87. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "ifeval-unspecified",
    "score": 0.8121,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'IFEval' (Strict), column 'Hermes 3 70B' = 81.21. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "ifeval-unspecified",
    "score": 0.6225,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'IFEval' (Strict), column 'Hermes 3 8B' = 62.25. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "mmlu",
    "score": 0.8502,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU' (5-shot), column 'Hermes 3 405B' = 85.02. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "mmlu",
    "score": 0.7909,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU' (5-shot), column 'Hermes 3 70B' = 79.09. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "mmlu",
    "score": 0.6479,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU' (5-shot), column 'Hermes 3 8B' = 64.79. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "mmlu-pro",
    "score": 0.5414,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU-PRO' (5-shot), column 'Hermes 3 405B' = 54.14. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "mmlu-pro",
    "score": 0.4724,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU-PRO' (5-shot), column 'Hermes 3 70B' = 47.24. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "mmlu-pro",
    "score": 0.3208,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MMLU-PRO' (5-shot), column 'Hermes 3 8B' = 32.08. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "musr",
    "score": 0.4826,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MuSR' (0-shot), column 'Hermes 3 405B' = 48.26. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "musr",
    "score": 0.5067,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MuSR' (0-shot), column 'Hermes 3 70B' = 50.67. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "musr",
    "score": 0.4352,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'MuSR' (0-shot), column 'Hermes 3 8B' = 43.52. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "openbookqa",
    "score": 0.488,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'OpenbookQA' (0-shot), column 'Hermes 3 405B' = 48.8. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "openbookqa",
    "score": 0.494,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'OpenbookQA' (0-shot), column 'Hermes 3 70B' = 49.4. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "openbookqa",
    "score": 0.478,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'OpenbookQA' (0-shot), column 'Hermes 3 8B' = 47.8. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "piqa",
    "score": 0.8596,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'PiQA' (0-shot), column 'Hermes 3 405B' = 85.96. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "piqa",
    "score": 0.8444,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'PiQA' (0-shot), column 'Hermes 3 70B' = 84.44. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "piqa",
    "score": 0.8025,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'PiQA' (0-shot), column 'Hermes 3 8B' = 80.25. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "truthfulqa-mc2",
    "score": 0.6557,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'TruthfulQA' (MC2 0-shot), column 'Hermes 3 405B' = 65.57. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "truthfulqa-mc2",
    "score": 0.6329,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'TruthfulQA' (MC2 0-shot), column 'Hermes 3 70B' = 63.29. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "truthfulqa-mc2",
    "score": 0.5869,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'TruthfulQA' (MC2 0-shot), column 'Hermes 3 8B' = 58.69. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "winogrande",
    "score": 0.8627,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Winogrande' (5-shot), column 'Hermes 3 405B' = 86.27. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "winogrande",
    "score": 0.8319,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Winogrande' (5-shot), column 'Hermes 3 70B' = 83.19. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-8b",
    "bench": "winogrande",
    "score": 0.7774,
    "source": "https://arxiv.org/pdf/2408.11857v1",
    "by": "lab",
    "date": "2024-08-15",
    "note": "Hermes 3 Technical Report, arXiv 2408.11857v1 (non-exclusive licence -> rule b), Table 5 'Final downstream task evaluations', row 'Winogrande' (5-shot), column 'Hermes 3 8B' = 77.74. Section 5: lm-evaluation-harness style evaluations from Section 4.1 plus Open LLM Leaderboard tasks; 'For the 405B models, evaluations were performed under FP8 quantization.' Vendor-reported, hand-recorded; the Llama 3.1 Instruct columns are not used.",
    "source_id": "first-party-nous-research-scores"
  },
  {
    "model": "hermes-3-llama-3-1-70b",
    "bench": "livebench-20240726",
    "score": 0.4005,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'hermes-3-llama-3.1-70b'. Release 2024-07-26 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26} and not removed by 2024-07-26; release dates from the question datasets livebench/<category> pinned at their 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Score computed by whatmodel.sh exactly as LiveBench's show_livebench_result.py (LiveBench/LiveBench @24364d6) does: mean per-question score per task → mean of task means per category → mean of the six categories = 40.05. Category means: math 28.3, coding 31.4, reasoning 33.3, language 43.8, instruction_following 55.4, data_analysis 48.1. Answers generated and scored by the LiveBench team ('We will evaluate your model!', README).",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "hermes-3-llama-3-1-405b",
    "bench": "aider-code-editing",
    "score": 0.639,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml",
    "by": "independent",
    "date": "2024-08-30",
    "note": "Published entry 'nousresearch/hermes-3-llama-3.1-405b'. Aider code editing benchmark (old edit leaderboard); 133 Exercism Python exercises; pass_rate_2 = 63.9 (percent completed correctly after at most two attempts, the leaderboard's headline column; not pass@1). edit_format=whole; Aider version=0.54.8-dev; evaluator commit=2d9d605; command=aider --model openrouter/nousresearch/hermes-3-llama-3.1-405b; run=2024-08-30-15-02-05--nous405b-whole. Identity: OpenRouter route 'nousresearch/hermes-3-llama-3.1-405b' (NousResearch/Hermes-3-Llama-3.1-405B as served by OpenRouter's providers; serving precision not stated). Apache-2.0 (repository LICENSE.txt).",
    "source_id": "aider-code-editing-runs"
  },
  {
    "model": "cogito-v1-preview-qwen-14b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9164,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-qwen-14B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v1-preview-qwen-14b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-qwen-14B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.8.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v1-preview-qwen-32b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9297,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-qwen-32B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v1-preview-qwen-32b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9285,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-qwen-32B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.8.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v1-preview-llama-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9189,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-70B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v1-preview-llama-70b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9174,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v1-preview-llama-70B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.9.1.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v2-preview-llama-70b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9248,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v2-preview-llama-70B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.15.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "cogito-v2-preview-llama-70b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9194,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepcogito/cogito-v2-preview-llama-70B', instruction-tuned (📝), default setting (EuroEval runs the standard chat template; Cogito hybrid models answer without the deep-thinking system prompt); EuroEval version 15.15.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "smollm2-360m",
    "bench": "ifeval-unspecified",
    "score": 0.41,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'IFEval (Average prompt/inst)', column 'SmolLM2-360M-Instruct' = 41.0; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "ifeval-unspecified",
    "score": 0.316,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "independent",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'IFEval (Average prompt/inst)', column 'Qwen2.5-0.5B-Instruct' = 31.6; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "smollm2-360m",
    "bench": "hellaswag",
    "score": 0.521,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'HellaSwag', column 'SmolLM2-360M-Instruct' = 52.1; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "hellaswag",
    "score": 0.48,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "independent",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'HellaSwag', column 'Qwen2.5-0.5B-Instruct' = 48.0; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "smollm2-360m",
    "bench": "piqa",
    "score": 0.708,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'PIQA', column 'SmolLM2-360M-Instruct' = 70.8; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "piqa",
    "score": 0.672,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "independent",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'PIQA', column 'Qwen2.5-0.5B-Instruct' = 67.2; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "smollm2-360m",
    "bench": "bbh",
    "score": 0.273,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'BBH (3-shot)', column 'SmolLM2-360M-Instruct' = 27.3; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "bbh",
    "score": 0.307,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "independent",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'BBH (3-shot)', column 'Qwen2.5-0.5B-Instruct' = 30.7; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "smollm2-360m",
    "bench": "gsm8k",
    "score": 0.0743,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "lab",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'GSM8K (5-shot)', column 'SmolLM2-360M-Instruct' = 7.43; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "qwen2-5-0-5b",
    "bench": "gsm8k",
    "score": 0.268,
    "source": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/blob/a10cc1512eabd3dde888204e902eca88bddb4951/README.md",
    "by": "independent",
    "date": "2025-09-22",
    "note": "SmolLM2-360M-Instruct model card (Apache-2.0, pinned a10cc15), instruction-model table, row 'GSM8K (5-shot)', column 'Qwen2.5-0.5B-Instruct' = 26.8; divided by 100. Card: 'All evaluations are zero-shot unless stated otherwise, and we use lighteval to run them' (Hugging Face's own runs of every column). MT-Bench, ARC (Average) and MMLU (cloze) rows not used (no matching id). SmolLM-360M-Instruct column not used (v1 instruct revision not stated).",
    "source_id": "card3-huggingfacetb-smollm2-360m-instruct"
  },
  {
    "model": "smollm2-360m",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2489,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'HuggingFaceTB/SmolLM2-360M-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "smollm2-135m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2477,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'HuggingFaceTB/SmolLM2-135M-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-350m",
    "bench": "mmlu",
    "score": 0.4343,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'MMLU' = 43.43; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "gpqa-unspecified",
    "score": 0.2746,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'GPQA' = 27.46; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "ifeval-unspecified",
    "score": 0.6512,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'IFEval' = 65.12; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "ifbench-unspecified",
    "score": 0.1641,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'IFBench' = 16.41; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "gsm8k",
    "score": 0.301,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'GSM8K' = 30.1; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "mgsm",
    "score": 0.2952,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'MGSM' = 29.52; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "mmmlu",
    "score": 0.3799,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-350M', column 'MMMLU' = 37.99; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "mmlu",
    "score": 0.499,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'MMLU' = 49.9; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "gpqa-unspecified",
    "score": 0.2848,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'GPQA' = 28.48; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "ifeval-unspecified",
    "score": 0.7223,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'IFEval' = 72.23; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "ifbench-unspecified",
    "score": 0.2056,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'IFBench' = 20.56; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "gsm8k",
    "score": 0.464,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'GSM8K' = 46.4; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "mgsm",
    "score": 0.4536,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'MGSM' = 45.36; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-700m",
    "bench": "mmmlu",
    "score": 0.4328,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-700M', column 'MMMLU' = 43.28; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "mmlu",
    "score": 0.5523,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'MMLU' = 55.23; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "gpqa-unspecified",
    "score": 0.3147,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'GPQA' = 31.47; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "ifeval-unspecified",
    "score": 0.7489,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'IFEval' = 74.89; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "ifbench-unspecified",
    "score": 0.207,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'IFBench' = 20.7; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "gsm8k",
    "score": 0.583,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'GSM8K' = 58.3; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "mgsm",
    "score": 0.5504,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'MGSM' = 55.04; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "mmmlu",
    "score": 0.4673,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'LFM2-1.2B', column 'MMMLU' = 46.73; divided by 100. Liquid AI's own LFM2 columns only (the card gives no evaluation statement for its Qwen3/Llama/Gemma comparator rows, so they are not used); MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-350m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.3781,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-350M', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-350m",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2557,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-350M', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-700m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.4209,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-700M', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-700m",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2506,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-700M', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6307,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-1.2B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2959,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-1.2B', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7633,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-2.6B', instruction-tuned (📝), default setting; EuroEval version 16.2.2.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.6451,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2-2.6B', instruction-tuned (📝), default setting; EuroEval version 16.2.2.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.6119,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2.5-1.2B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.325,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'LiquidAI/LFM2.5-1.2B-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "gpqa-diamond",
    "score": 0.3889,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'GPQA Diamond' = 38.89; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "mmlu-pro",
    "score": 0.4435,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'MMLU-Pro' = 44.35; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "ifeval-mean-four",
    "score": 0.8623,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'IFEval' = 86.23; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "ifbench-unspecified",
    "score": 0.4733,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'IFBench' = 47.33; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "gsm8k",
    "score": 0.6452,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'GSM8K' = 64.52; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "math-500",
    "score": 0.632,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'MATH-500' = 63.20; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "aime-2025",
    "score": 0.14,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'AIME25' = 14.00; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "lfm2-5-1-2b-instruct",
    "bench": "bfcl-v3",
    "score": 0.4912,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "lab",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'LFM2.5-1.2B-Instruct', column 'BFCLv3' = 49.12; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "gpqa-diamond",
    "score": 0.2434,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'GPQA Diamond' = 24.34; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "mmlu-pro",
    "score": 0.2764,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'MMLU-Pro' = 27.64; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "ifeval-mean-four",
    "score": 0.8008,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'IFEval' = 80.08; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "ifbench-unspecified",
    "score": 0.2493,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'IFBench' = 24.93; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "gsm8k",
    "score": 0.696,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'GSM8K' = 69.60; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "math-500",
    "score": 0.472,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'MATH-500' = 47.20; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "aime-2025",
    "score": 0.01,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'AIME25' = 1; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "granite-4-0-h-1b",
    "bench": "bfcl-v3",
    "score": 0.5069,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/f313478934a7612d22991f752959d7a1a8756fec/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "Liquid AI's own comparison run in the pinned LFM2.5-1.2B-Thinking card Benchmarks table, row 'Granite-4.0-H-1B', column 'BFCLv3' = 50.69; divided by 100. Same table, mapping and methodology as the existing rows of this source (IFEval = mean of strict/loose prompt/instruction accuracy; BFCLv3 final weighted average with Liquid's custom handler; GPQA/MMLU-Pro/IFBench/AIME25 Liquid's own runs following the cited methodology; instruct models with greedy decoding). Multi-IF not used. Date = pinned card revision.",
    "source_id": "card-liquidai-lfm2-5-1-2b-thinking"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "puredocbench-avg3",
    "score": 0.7763,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.8-27B (2026-08)', Overall (Avg3) = 77.63 (0-100) (Qwen/Qwen3.8-27B); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "puredocbench-avg3",
    "score": 0.6854,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.8-Flash-Next (2026-08)', Overall (Avg3) = 68.54 (0-100) (Qwen/Qwen3.8-Flash-Next); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "puredocbench-avg3",
    "score": 0.6714,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.6-35B-A3B (2026-04)', Overall (Avg3) = 67.14 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "puredocbench-avg3",
    "score": 0.6562,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.6-27B (2026-04)', Overall (Avg3) = 65.62 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "puredocbench-avg3",
    "score": 0.6672,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-397B-A17B (2026-02)', Overall (Avg3) = 66.72 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "puredocbench-avg3",
    "score": 0.7411,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-122B-A10B (2026-02)', Overall (Avg3) = 74.11 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "puredocbench-avg3",
    "score": 0.6568,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-35B-A3B (2026-02)', Overall (Avg3) = 65.68 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "puredocbench-avg3",
    "score": 0.6957,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-27B (2026-02)', Overall (Avg3) = 69.57 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "puredocbench-avg3",
    "score": 0.7089,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-9B (2026-03)', Overall (Avg3) = 70.89 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "puredocbench-avg3",
    "score": 0.6982,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-4B (2026-03)', Overall (Avg3) = 69.82 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "puredocbench-avg3",
    "score": 0.6246,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-2B (2026-03)', Overall (Avg3) = 62.46 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "puredocbench-avg3",
    "score": 0.5599,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3.5-0.8B (2026-03)', Overall (Avg3) = 55.99 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "puredocbench-avg3",
    "score": 0.6907,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3-VL-8B (2025-10)', Overall (Avg3) = 69.07 (0-100) (README links Qwen/Qwen3-VL-8B-Instruct); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "puredocbench-avg3",
    "score": 0.675,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3-VL-4B (2025-10)', Overall (Avg3) = 67.5 (0-100) (README links Qwen/Qwen3-VL-4B-Instruct); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "puredocbench-avg3",
    "score": 0.6209,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Qwen3-VL-2B (2025-10)', Overall (Avg3) = 62.09 (0-100) (README links Qwen/Qwen3-VL-2B-Instruct); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "deepseek-v4-flash-vision-exp",
    "bench": "puredocbench-avg3",
    "score": 0.5012,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'DeepSeek-V4-Flash-Vision-Exp (2026-08)', Overall (Avg3) = 50.12 (0-100) (deepseek-ai/DeepSeek-V4-Flash-Vision-Exp); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "puredocbench-avg3",
    "score": 0.7969,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'GLM-5.3-Flash (2026-08)', Overall (Avg3) = 79.69 (0-100) (zai-org/GLM-5.3-Flash); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "puredocbench-avg3",
    "score": 0.7827,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Gemini 3.6 Flash (2026-07)', Overall (Avg3) = 78.27 (0-100) (Gemini API gemini-3.6-flash); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "puredocbench-avg3",
    "score": 0.7043,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Gemini-3.1-Pro (2026-02)', Overall (Avg3) = 70.43 (0-100) (Gemini API gemini-3.1-pro-preview); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "claude-opus-5",
    "bench": "puredocbench-avg3",
    "score": 0.7707,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Claude Opus 5 (2026-07)', Overall (Avg3) = 77.07 (0-100); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "puredocbench-avg3",
    "score": 0.6916,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'GPT-5.6 Sol (2026-07)', Overall (Avg3) = 69.16 (0-100) (OpenAI gpt-5.6-sol); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "minimax-m3",
    "bench": "puredocbench-avg3",
    "score": 0.5072,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'MiniMax-M3 (2026-06)', Overall (Avg3) = 50.72 (0-100) (MiniMaxAI/MiniMax-M3); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "kimi-k2-6",
    "bench": "puredocbench-avg3",
    "score": 0.701,
    "source": "https://arxiv.org/html/2605.07492v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0) Table 2 (three-track parsing accuracy for 58 models), General-purpose VLMs block, row 'Kimi K2.6 (2026-04)', Overall (Avg3) = 70.1 (0-100) (moonshotai/Kimi-K2.6); stored /100. Authors' own runs on the new benchmark (open models served locally, closed models via their APIs; prompts and predictions released); checkpoint per the authors' README @154f576. Reasoning settings for API models not stated.",
    "source_id": "paper-2605.07492v2-puredocbench"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.3894,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'Opus 4.7', Pass@1 = 38.94% (thinking xHigh); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "gpt-5-4",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.38,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'GPT 5.4', Pass@1 = 38.0% (thinking xHigh); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.3183,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'Opus 4.6', Pass@1 = 31.83% (thinking High); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.2373,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'Gemini 3.1 Pro', Pass@1 = 23.73% (thinking High); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "glm-5",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.2403,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'GLM 5', Pass@1 = 24.03% (default (High)); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "kimi-k2-5",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.1905,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'Kimi K2.5', Pass@1 = 19.05% (default (High)); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "minimax-m2-5",
    "bench": "swe-atlas-mini-swe-agent",
    "score": 0.152,
    "source": "https://arxiv.org/html/2605.08366v1",
    "by": "independent",
    "date": "2026-05-08",
    "note": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0) Table 1 (resolution rates on SWE Atlas), mini-SWE-Agent block, row 'Minimax M2.5', Pass@1 = 15.2% (default (High)); stored /100. Authors' own run: Harbor framework, mini-SWE-agent scaffold, 3 trials per task, thinking setting per Table 8.",
    "source_id": "paper-2605.08366v1-swe-atlas"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "diagnosticiq-macro",
    "score": 0.7359,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'claude-opus-4-6', Macro % D.IQ = 73.59%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gpt-5-4",
    "bench": "diagnosticiq-macro",
    "score": 0.7327,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gpt-5.4', Macro % D.IQ = 73.27%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "diagnosticiq-macro",
    "score": 0.7274,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gemini-3.1-pro-pre', Macro % D.IQ = 72.74% (Gemini 3.1 Pro Preview); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "diagnosticiq-macro",
    "score": 0.6484,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'claude-4-6-sonnet', Macro % D.IQ = 64.84%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "diagnosticiq-macro",
    "score": 0.7061,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'claude-3-7-sonnet', Macro % D.IQ = 70.61%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "diagnosticiq-macro",
    "score": 0.699,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'Qwen3.6-35B', Macro % D.IQ = 69.9% (Qwen3.6-35B-A3B); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gpt-5",
    "bench": "diagnosticiq-macro",
    "score": 0.6589,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gpt-5-2025-08-07', Macro % D.IQ = 65.89%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "diagnosticiq-macro",
    "score": 0.6541,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'o1', Macro % D.IQ = 65.41% (API alias o1 = o1-2024-12-17, the only o1 snapshot); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gemma-4-31b",
    "bench": "diagnosticiq-macro",
    "score": 0.6506,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gemma-4-31b', Macro % D.IQ = 65.06%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "llama-4-maverick",
    "bench": "diagnosticiq-macro",
    "score": 0.6329,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'llama-4-maverick', Macro % D.IQ = 63.29%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "diagnosticiq-macro",
    "score": 0.632,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gemma-4-26b', Macro % D.IQ = 63.2%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "diagnosticiq-macro",
    "score": 0.6252,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'claude-4-sonnet', Macro % D.IQ = 62.52%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "diagnosticiq-macro",
    "score": 0.6167,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'llama-3-3-70b-ins.', Macro % D.IQ = 61.67%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "diagnosticiq-macro",
    "score": 0.6122,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'qwen2-5-72b-ins.', Macro % D.IQ = 61.22%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "diagnosticiq-macro",
    "score": 0.6117,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'mistral-s-3-1-24b', Macro % D.IQ = 61.17% (Mistral Small 3.1 24B); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "mistral-medium-2505",
    "bench": "diagnosticiq-macro",
    "score": 0.6034,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'mistral-m-2505', Macro % D.IQ = 60.34%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "diagnosticiq-macro",
    "score": 0.5945,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "lab",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'granite-3-3-8b-ins.', Macro % D.IQ = 59.45% (IBM's own model (by: lab)); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "diagnosticiq-macro",
    "score": 0.5764,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gemini-2.0-flash', Macro % D.IQ = 57.64% (GA alias gemini-2.0-flash = 001); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "diagnosticiq-macro",
    "score": 0.5759,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'gemini-2.5-pro', Macro % D.IQ = 57.59%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "diagnosticiq-macro",
    "score": 0.5656,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'llama-3-1-405b', Macro % D.IQ = 56.56% (Instruct assumed); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "granite-4-0-h-small",
    "bench": "diagnosticiq-macro",
    "score": 0.5316,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "lab",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'granite-4-h-small', Macro % D.IQ = 53.16% (IBM's own model (by: lab)); stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "phi-4",
    "bench": "diagnosticiq-macro",
    "score": 0.5052,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'microsoft-phi-4', Macro % D.IQ = 50.52%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "diagnosticiq-macro",
    "score": 0.4693,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'claude-3-5-haiku', Macro % D.IQ = 46.93%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "qwen3-8b",
    "bench": "diagnosticiq-macro",
    "score": 0.4621,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'qwen3-8b', Macro % D.IQ = 46.21%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "llama-3-1-8b",
    "bench": "diagnosticiq-macro",
    "score": 0.3869,
    "source": "https://arxiv.org/html/2605.08614v1",
    "by": "independent",
    "date": "2026-05-09",
    "note": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0) Table 2 (leaderboard), row 'llama-3-1-8b-ins.', Macro % D.IQ = 38.69%; stored /100. Authors' own run: zero-shot direct prompting, temperature 0, max tokens 4,096, other parameters at provider defaults.",
    "source_id": "paper-2605.08614v1-diagnosticiq"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "wildclawbench",
    "score": 0.622,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Claude Opus 4.7', Overall Score = 62.2%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "gpt-5-5",
    "bench": "wildclawbench",
    "score": 0.582,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'GPT 5.5', Overall Score = 58.2%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "wildclawbench",
    "score": 0.516,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Claude Opus 4.6', Overall Score = 51.6%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "gpt-5-4",
    "bench": "wildclawbench",
    "score": 0.503,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'GPT 5.4', Overall Score = 50.3%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "glm-5-1",
    "bench": "wildclawbench",
    "score": 0.482,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'GLM 5.1', Overall Score = 48.2%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "wildclawbench",
    "score": 0.437,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'DeepSeek V4 Pro', Overall Score = 43.7% (paper dated 2026-05-11, so the April 2026 V4 Pro release); stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "wildclawbench",
    "score": 0.43,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'MiMo V2.5 Pro', Overall Score = 43.0%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "glm-5",
    "bench": "wildclawbench",
    "score": 0.426,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'GLM 5', Overall Score = 42.6%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "wildclawbench",
    "score": 0.408,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Gemini 3.1 Pro', Overall Score = 40.8%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "wildclawbench",
    "score": 0.402,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'MiMo V2 Pro', Overall Score = 40.2%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "wildclawbench",
    "score": 0.345,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Qwen3.5 397B', Overall Score = 34.5%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "wildclawbench",
    "score": 0.34,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'DeepSeek V3.2', Overall Score = 34.0%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "minimax-m2-7",
    "bench": "wildclawbench",
    "score": 0.338,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'MiniMax M2.7', Overall Score = 33.8%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "kimi-k2-5",
    "bench": "wildclawbench",
    "score": 0.308,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Kimi K2.5', Overall Score = 30.8%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "mimo-v2-flash",
    "bench": "wildclawbench",
    "score": 0.308,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'MiMo V2 Flash', Overall Score = 30.8%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "minimax-m2-5",
    "bench": "wildclawbench",
    "score": 0.271,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'MiniMax M2.5', Overall Score = 27.1%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "step-3-5-flash",
    "bench": "wildclawbench",
    "score": 0.267,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Step 3.5 Flash', Overall Score = 26.7%; stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "grok-4-20",
    "bench": "wildclawbench",
    "score": 0.193,
    "source": "https://arxiv.org/html/2605.10912v1",
    "by": "independent",
    "date": "2026-05-11",
    "note": "WildClawBench (arXiv:2605.10912v1, CC BY 4.0) Table 2 (main results on WildClawBench under the OpenClaw harness), row 'Grok 4.20 Beta', Overall Score = 19.3% (cited as grok-4.20-0309-reasoning); stored /100. Authors' own run on the benchmark they introduce: OpenClaw harness in a pinned Docker image, models served through one OpenRouter endpoint, 60 long-horizon tasks (26 multimodal + 34 pure text), Overall = task-count-weighted average.",
    "source_id": "paper-2605.10912v1-wildclawbench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "contractbench-sr",
    "score": 0.778,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Claude Opus 4.6', SR% = 77.8% (77/99; pinned Anthropic claude-opus-4-6); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "contractbench-sr",
    "score": 0.697,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Claude Sonnet 4.5', SR% = 69.7% (69/99; pinned claude-sonnet-4-5-20250929); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "contractbench-sr",
    "score": 0.515,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemini 2.5 Pro', SR% = 51.5% (51/99; Google AI Studio gemini-2.5-pro); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "contractbench-sr",
    "score": 0.192,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemini 2.5 Flash', SR% = 19.2% (19/99; Google AI Studio gemini-2.5-flash); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gpt-5-2",
    "bench": "contractbench-sr",
    "score": 0.747,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'GPT-5.2', SR% = 74.7% (74/99; OpenAI gpt-5.2); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gpt-5-1",
    "bench": "contractbench-sr",
    "score": 0.485,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'GPT-5.1', SR% = 48.5% (48/99; OpenAI gpt-5.1); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gpt-5",
    "bench": "contractbench-sr",
    "score": 0.707,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'GPT-5', SR% = 70.7% (70/99; OpenAI gpt-5); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "contractbench-sr",
    "score": 0.232,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'GPT-4o', SR% = 23.2% (23/99; pinned gpt-4o-2024-08-06); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "contractbench-sr",
    "score": 0.707,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Qwen3.5-397B-A17B (MoE)', SR% = 70.7% (70/99; Qwen/Qwen3.5-397B-A17B); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "contractbench-sr",
    "score": 0.646,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Qwen3.5-27B', SR% = 64.6% (64/99; Qwen/Qwen3.5-27B); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "contractbench-sr",
    "score": 0.566,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Qwen3.5-9B', SR% = 56.6% (56/99; Qwen/Qwen3.5-9B, Instruct block); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "contractbench-sr",
    "score": 0.232,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Qwen2.5-72B-Instruct', SR% = 23.2% (23/99; Qwen/Qwen2.5-72B-Instruct); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "minimax-m2-5",
    "bench": "contractbench-sr",
    "score": 0.626,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'MiniMax-M2.5', SR% = 62.6% (62/99; Together MiniMaxAI/MiniMax-M2.5); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "minimax-m2-1",
    "bench": "contractbench-sr",
    "score": 0.606,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'MiniMax-M2.1', SR% = 60.6% (60/99; MiniMaxAI/MiniMax-M2.1); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "minimax-m2",
    "bench": "contractbench-sr",
    "score": 0.535,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'MiniMax-M2', SR% = 53.5% (53/99; MiniMaxAI/MiniMax-M2); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "mistral-small-4",
    "bench": "contractbench-sr",
    "score": 0.424,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Mistral-Small-4 (MoE)', SR% = 42.4% (42/99; OpenRouter mistralai/mistral-small-2603); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "ministral-3-14b",
    "bench": "contractbench-sr",
    "score": 0.283,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Ministral-3-14B', SR% = 28.3% (28/99; OpenRouter mistralai/ministral-14b-2512 (Instruct)); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "ministral-3-8b",
    "bench": "contractbench-sr",
    "score": 0.192,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Ministral-3-8B', SR% = 19.2% (19/99; OpenRouter mistralai/ministral-8b-2512 (Instruct)); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "ministral-3-3b",
    "bench": "contractbench-sr",
    "score": 0.061,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Ministral-3-3B', SR% = 6.1% (6/99; OpenRouter mistralai/ministral-3b-2512 (Instruct)); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemma-4-31b",
    "bench": "contractbench-sr",
    "score": 0.374,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemma-4-31B', SR% = 37.4% (37/99; google/gemma-4-31B-it); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "contractbench-sr",
    "score": 0.384,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemma-4-26B-A4B (MoE)', SR% = 38.4% (38/99; OpenRouter google/gemma-4-26b-a4b-it); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "contractbench-sr",
    "score": 0.172,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemma-4-E4B', SR% = 17.2% (17/99; Ollama gemma-4-e4b); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "contractbench-sr",
    "score": 0.071,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Gemma-4-E2B', SR% = 7.1% (7/99; Ollama (Instruct)); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "contractbench-sr",
    "score": 0.071,
    "source": "https://arxiv.org/html/2605.17281v2",
    "by": "independent",
    "date": "2026-09-27",
    "note": "ContractBench (arXiv:2605.17281v2, CC BY 4.0) Table 10 (master leaderboard, 38 model variants x 33 tasks, k=3, n=99), row 'Llama-3.3-70B-Instruct', SR% = 7.1% (7/99; Together meta-llama/Llama-3.3-70B-Instruct-Turbo); stored /100. Authors' own runs: temperature 0, k=3 rollouts per task, 600 s per-episode timeout, provider/backend as printed in Table 10 and pinned identifier from Table 9; reasoning effort not stated (provider default).",
    "source_id": "paper-2605.17281v2-contractbench"
  },
  {
    "model": "gpt-5",
    "bench": "chartarena-en-map",
    "score": 0.456,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'GPT-5 (Table 2)', Average EN (mAP_high) = 45.6 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "chartarena-en-map",
    "score": 0.477,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Gemini 2.5 Pro (Table 2)', Average EN (mAP_high) = 47.7 (0-100) (Gemini API; release column 2025.03 is the family date, GA endpoint assumed); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "chartarena-en-map",
    "score": 0.592,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Gemini 3.1 Pro (Table 2)', Average EN (mAP_high) = 59.2 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen2-5-vl-72b",
    "bench": "chartarena-en-map",
    "score": 0.366,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen2.5-VL-72B-Ins. (Table 2)', Average EN (mAP_high) = 36.6 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen2-5-vl-32b",
    "bench": "chartarena-en-map",
    "score": 0.318,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen2.5-VL-32B-Ins. (Table C.3)', Average EN (mAP_high) = 31.8 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen2-5-vl-7b",
    "bench": "chartarena-en-map",
    "score": 0.233,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen2.5-VL-7B-Ins. (Table 2)', Average EN (mAP_high) = 23.3 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen2-5-vl-3b",
    "bench": "chartarena-en-map",
    "score": 0.214,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen2.5-VL-3B-Ins. (Table C.3)', Average EN (mAP_high) = 21.4 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-235b-a22b-instruct",
    "bench": "chartarena-en-map",
    "score": 0.46,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-235B-A22B-Ins. (Table 2)', Average EN (mAP_high) = 46.0 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "chartarena-en-map",
    "score": 0.373,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-8B-Ins. (Table 2)', Average EN (mAP_high) = 37.3 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "chartarena-en-map",
    "score": 0.396,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-8B-Think. (Table C.3)', Average EN (mAP_high) = 39.6 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-4b-instruct",
    "bench": "chartarena-en-map",
    "score": 0.332,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-4B-Ins. (Table C.3)', Average EN (mAP_high) = 33.2 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "chartarena-en-map",
    "score": 0.347,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-4B-Think. (Table C.3)', Average EN (mAP_high) = 34.7 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "chartarena-en-map",
    "score": 0.27,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-2B-Ins. (Table C.3)', Average EN (mAP_high) = 27.0 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-vl-2b-thinking",
    "bench": "chartarena-en-map",
    "score": 0.265,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3-VL-2B-Think. (Table C.3)', Average EN (mAP_high) = 26.5 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "chartarena-en-map",
    "score": 0.541,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3.5-35B-A3B (think) (Table 2)', Average EN (mAP_high) = 54.1 (0-100) (thinking mode); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "chartarena-en-map",
    "score": 0.406,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3.5-9B (Table C.3)', Average EN (mAP_high) = 40.6 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "chartarena-en-map",
    "score": 0.335,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3.5-4B (Table C.3)', Average EN (mAP_high) = 33.5 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "chartarena-en-map",
    "score": 0.162,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3.5-2B (Table C.3)', Average EN (mAP_high) = 16.2 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "chartarena-en-map",
    "score": 0.098,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Qwen3.5-0.8B (Table C.3)', Average EN (mAP_high) = 9.8 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "glm-4-5v",
    "bench": "chartarena-en-map",
    "score": 0.438,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'GLM-4.5V (Table 2)', Average EN (mAP_high) = 43.8 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "kimi-k2-5",
    "bench": "chartarena-en-map",
    "score": 0.548,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Kimi K2.5 (no-think) (Table 2)', Average EN (mAP_high) = 54.8 (0-100) (non-thinking mode, the only setting reported); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "chartarena-en-map",
    "score": 0.439,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'MiMo-V2-Omni (Table 2)', Average EN (mAP_high) = 43.9 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "gemma-4-31b",
    "bench": "chartarena-en-map",
    "score": 0.468,
    "source": "https://arxiv.org/html/2606.01348v3",
    "by": "independent",
    "date": "2026-07-05",
    "note": "ChartArena (arXiv:2606.01348v3, CC BY 4.0) Table 2 (main results) / Table C.3 (additional results), row 'Gemma 4-31B-IT (Table C.3)', Average EN (mAP_high) = 46.8 (0-100); stored /100. Authors' own run: unified chart-parsing prompt for general-purpose MLLMs, identical inference settings, outputs normalised before scoring; reasoning mode as printed in the row label.",
    "source_id": "paper-2606.01348v3-chartarena"
  },
  {
    "model": "gpt-5-5",
    "bench": "k-browsecomp-verified",
    "score": 0.4567,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'GPT-5.5', Verified Pass@1 Acc. = 45.67%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "k-browsecomp-verified",
    "score": 0.3067,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'GPT-5.4-mini', Verified Pass@1 Acc. = 30.67%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "glm-5-1",
    "bench": "k-browsecomp-verified",
    "score": 0.3067,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'GLM-5.1', Verified Pass@1 Acc. = 30.67%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "k-browsecomp-verified",
    "score": 0.3,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'DeepSeek-V4-Pro', Verified Pass@1 Acc. = 30.0% (paper dated 2026-06-01, so the April 2026 V4 Pro release); stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "gemma-4-31b",
    "bench": "k-browsecomp-verified",
    "score": 0.2333,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'Gemma-4-31B-it', Verified Pass@1 Acc. = 23.33%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "qwen3-6-35b-a3b",
    "bench": "k-browsecomp-verified",
    "score": 0.12,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'Qwen3.6-35B-A3B', Verified Pass@1 Acc. = 12.0%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "gemini-3-1-flash-lite",
    "bench": "k-browsecomp-verified",
    "score": 0.1133,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'Gemini-3.1-Flash-Lite', Verified Pass@1 Acc. = 11.33%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "k-exaone-236b-a23b",
    "bench": "k-browsecomp-verified",
    "score": 0.1033,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "independent",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'K-EXAONE-236B-A23B', Verified Pass@1 Acc. = 10.33%; stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "hyperclova-x-seed-32b-think",
    "bench": "k-browsecomp-verified",
    "score": 0.0233,
    "source": "https://arxiv.org/html/2606.02404v1",
    "by": "lab",
    "date": "2026-06-01",
    "note": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0) Table 2 (performance on K-BrowseComp-Verified and the synthetic split), row 'HCX-SEED-Think-32B', Verified Pass@1 Acc. = 2.33% (HyperCLOVAX-SEED-Think-32B; NAVER Cloud co-author (by: lab)); stored /100. Authors' own run: search_evals deep-research agent, Perplexity Search backend, 10 search calls per question, one run per question, GPT-5.4-mini answer extraction.",
    "source_id": "paper-2606.02404v1-k-browsecomp"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "kina-accuracy",
    "score": 0.5317,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Gemini-3.1-Pro-Preview', ALL = 53.17%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "kina-accuracy",
    "score": 0.4992,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Claude-Opus-4.6', ALL = 49.92%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "gpt-5-4",
    "bench": "kina-accuracy",
    "score": 0.4855,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'GPT-5.4', ALL = 48.55%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "kina-accuracy",
    "score": 0.4391,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Gemini-3-Flash-Preview', ALL = 43.91%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "kina-accuracy",
    "score": 0.4299,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-397B-A17B', ALL = 42.99%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "kimi-k2-5",
    "bench": "kina-accuracy",
    "score": 0.4024,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Kimi-K2.5', ALL = 40.24%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "gpt-5-2",
    "bench": "kina-accuracy",
    "score": 0.3952,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'GPT-5.2', ALL = 39.52%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "kina-accuracy",
    "score": 0.3935,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-27B', ALL = 39.35%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "kina-accuracy",
    "score": 0.3888,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-122B-A10B', ALL = 38.88% (Table 13: Qwen3.5-122B-A10B-FP8 checkpoint); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "kina-accuracy",
    "score": 0.3801,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'DeepSeek-V3.2-Thinking', ALL = 38.01% (thinking mode); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-max-2025-09-23",
    "bench": "kina-accuracy",
    "score": 0.359,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-Max-2025-09-23', ALL = 35.9% (Table 13: Qwen3-Max-250923); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "glm-5",
    "bench": "kina-accuracy",
    "score": 0.3585,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'GLM-5', ALL = 35.85%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "kina-accuracy",
    "score": 0.3543,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-35B-A3B', ALL = 35.43% (Table 13: Qwen3.5-35B-A3B-FP8 checkpoint); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "kina-accuracy",
    "score": 0.3373,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Grok-4.1-Fast-Reasoning', ALL = 33.73%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "kina-accuracy",
    "score": 0.3215,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-235B-A22B-Thinking-2507', ALL = 32.15%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "llama-4-maverick",
    "bench": "kina-accuracy",
    "score": 0.3162,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Llama-4-Maverick-17B', ALL = 31.62% (Table 13: Llama-4-Maverick-17B-Instruct-FP8); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "minimax-m2-5",
    "bench": "kina-accuracy",
    "score": 0.3028,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Minimax-M2.5', ALL = 30.28%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "kina-accuracy",
    "score": 0.3009,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-9B', ALL = 30.09%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-next-80b-a3b-instruct",
    "bench": "kina-accuracy",
    "score": 0.3009,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-Next-80B-A3B-Instruct', ALL = 30.09%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "kina-accuracy",
    "score": 0.3001,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Claude-Sonnet-4.6', ALL = 30.01%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "kina-accuracy",
    "score": 0.2959,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Llama-3.1-405B-Instruct', ALL = 29.59%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "kina-accuracy",
    "score": 0.2937,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-235B-A22B', ALL = 29.37% (original April-2025 hybrid release); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "step-3-5-flash",
    "bench": "kina-accuracy",
    "score": 0.2912,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Step-3.5-Flash', ALL = 29.12%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "kina-accuracy",
    "score": 0.285,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-4B', ALL = 28.5%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-next-80b-a3b-thinking",
    "bench": "kina-accuracy",
    "score": 0.2828,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-Next-80B-A3B-Thinking', ALL = 28.28% (Table 13: Qwen3-Next-80B-A3B-Thinking-FP8); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-32b",
    "bench": "kina-accuracy",
    "score": 0.2741,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-32B', ALL = 27.41%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "kina-accuracy",
    "score": 0.2703,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-30B-A3B-Thinking-2507', ALL = 27.03%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "llama-3-70b",
    "bench": "kina-accuracy",
    "score": 0.2661,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Llama-3-70B-Instruct', ALL = 26.61%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-14b",
    "bench": "kina-accuracy",
    "score": 0.26,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-14B', ALL = 26.0%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen2-72b",
    "bench": "kina-accuracy",
    "score": 0.2492,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen2-72B-Instruct', ALL = 24.92%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "kina-accuracy",
    "score": 0.2483,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-4B-Thinking-2507', ALL = 24.83%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "kina-accuracy",
    "score": 0.2428,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-30B-A3B', ALL = 24.28% (original April-2025 hybrid release); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "kina-accuracy",
    "score": 0.2303,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen2.5-72B-Instruct', ALL = 23.03%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-8b",
    "bench": "kina-accuracy",
    "score": 0.2227,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-8B', ALL = 22.27%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-4b",
    "bench": "kina-accuracy",
    "score": 0.215,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-4B', ALL = 21.5%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "kina-accuracy",
    "score": 0.2052,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-2B', ALL = 20.52%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "kina-accuracy",
    "score": 0.1833,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-1.7B', ALL = 18.33%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "kina-accuracy",
    "score": 0.1783,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Mixtral-8x7B-Instruct', ALL = 17.83% (Table 13: Mixtral-8x7B-Instruct-v0.1); stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "kina-accuracy",
    "score": 0.1666,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3.5-0.8B', ALL = 16.66%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "kina-accuracy",
    "score": 0.1449,
    "source": "https://arxiv.org/html/2606.05104v2",
    "by": "independent",
    "date": "2026-06-04",
    "note": "KINA (arXiv:2606.05104v2, CC BY 4.0) Table 3 (full leaderboard, 42 models), row 'Qwen3-0.6B', ALL = 14.49%; stored /100. Authors' own run: avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K / 'Medium', zero-shot, no tools; model identifiers per Table 13.",
    "source_id": "paper-2606.05104v2-kina"
  },
  {
    "model": "qwen2-5-72b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.68,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Qwen2.5-72B-Instruct', Avg. (micro-accuracy) = 68.0% (Qwen/Qwen2.5-72B-Instruct); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.6492,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Qwen2.5-32B-Instruct', Avg. (micro-accuracy) = 64.92% (Qwen/Qwen2.5-32B-Instruct); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.6463,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'GPT-OSS-120B', Avg. (micro-accuracy) = 64.63% (openai/gpt-oss-120b, log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.5756,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'GPT-OSS-20B', Avg. (micro-accuracy) = 57.56% (openai/gpt-oss-20b, log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "ministral-3-14b-reasoning",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.6348,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Ministral-3-14B-Reasoning', Avg. (micro-accuracy) = 63.48% (log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "ministral-3-8b-reasoning",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.5845,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Ministral-3-8B-Reasoning', Avg. (micro-accuracy) = 58.45% (log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "ministral-3-8b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.544,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Ministral-3-8B-Instruct', Avg. (micro-accuracy) = 54.4%; stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.6037,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'DeepSeek-R1-Distill-Qwen-32B', Avg. (micro-accuracy) = 60.37% (log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.5592,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'DeepSeek-R1-Distill-Qwen-14B', Avg. (micro-accuracy) = 55.92% (log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.3402,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'DeepSeek-R1-Distill-Qwen-7B', Avg. (micro-accuracy) = 34.02% (log-prob scoring); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "huatuogpt-o1-70b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.6545,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'HuatuoGPT-o1-70B', Avg. (micro-accuracy) = 65.45% (FreedomIntelligence/HuatuoGPT-o1-70B); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "huatuogpt-o1-7b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.5792,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'HuatuoGPT-o1-7B', Avg. (micro-accuracy) = 57.92% (FreedomIntelligence/HuatuoGPT-o1-7B); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "meditron-70b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.565,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Meditron-70B', Avg. (micro-accuracy) = 56.5% (epfl-llm/meditron-70b); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "meditron-7b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.2714,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Meditron-7B', Avg. (micro-accuracy) = 27.14% (epfl-llm/meditron-7b); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "biomistral-7b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.4714,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'BioMistral-7B', Avg. (micro-accuracy) = 47.14% (BioMistral/BioMistral-7B); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "medgemma-4b-it",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.4408,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'MedGemma-4B-IT', Avg. (micro-accuracy) = 44.08% (google/medgemma-4b-it); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "gemma-2-9b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.4387,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Gemma-2-9B-IT', Avg. (micro-accuracy) = 43.87% (google/gemma-2-9b-it); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "gemma-2-2b",
    "bench": "surgiq-zeroshot-logprob",
    "score": 0.3497,
    "source": "https://arxiv.org/html/2606.08071v1",
    "by": "independent",
    "date": "2026-06-06",
    "note": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0) Table 4 (zero-shot performance after duplicate filtering), row 'Gemma-2-2B-IT', Avg. (micro-accuracy) = 34.97% (google/gemma-2-2b-it); stored /100. Authors' own run: zero-shot fixed prompt, next-token log-probability over option letters (max over plain and space-prefixed labels), fixed shuffled option order, deterministic; reasoning models scored without generated reasoning.",
    "source_id": "paper-2606.08071v1-surgiq"
  },
  {
    "model": "gemma-4-31b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.4114,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Gemma4-31B-it', Pass^k (K=3) = 41.14%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gemma-4-26b-a4b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.3771,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Gemma4-26B-A4B-it', Pass^k (K=3) = 37.71% (also the user simulator and judge model); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.3562,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Gemma4-E4B-it', Pass^k (K=3) = 35.62%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.2838,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Gemma4-E2B-it', Pass^k (K=3) = 28.38%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.2933,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'GPT-OSS-120B', Pass^k (K=3) = 29.33% (reasoning effort high); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.0857,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'GPT-OSS-20B', Pass^k (K=3) = 8.57% (reasoning effort high); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gpt-5-4",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.2095,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'GPT-5.4', Pass^k (K=3) = 20.95% (reasoning effort low (only setting run)); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.141,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'GPT-5.4-Mini', Pass^k (K=3) = 14.1% (reasoning effort low (only setting run)); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.0724,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'GPT-5.4-Nano', Pass^k (K=3) = 7.24% (reasoning effort low (only setting run)); stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.3448,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Claude Opus 4.6', Pass^k (K=3) = 34.48%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.3371,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Claude Sonnet 4.5', Pass^k (K=3) = 33.71%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "t1-bench-pass-hat-3",
    "score": 0.3429,
    "source": "https://arxiv.org/html/2606.11070v1",
    "by": "independent",
    "date": "2026-06-09",
    "note": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0) Table 2 (overall results averaged over all domains), row 'Claude Haiku 4.5', Pass^k (K=3) = 34.29%; stored /100. Authors' own run: temperature 1.0, three trials per conversation, Gemma4-26B-A4B-it as simulated user; reasoning effort per Appendix Table 6.",
    "source_id": "paper-2606.11070v1-t1-bench"
  },
  {
    "model": "bielik-11b-v3-0-instruct",
    "bench": "polish-medical-exams-om",
    "score": 0.558,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Bielik-11B', Avg OM = 55.8% (Sec. 6.1: Bielik-11B-v3-Instruct); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "bielik-4-5b-v3-0-instruct",
    "bench": "polish-medical-exams-om",
    "score": 0.378,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Bielik-4.5B', Avg OM = 37.8% (Sec. 6.1: Bielik-4.5B-v3.0-Instruct); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "llama-pllum-70b-instruct-2412",
    "bench": "polish-medical-exams-om",
    "score": 0.571,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'PLLuM-70B-Inst', Avg OM = 57.1% (Sec. 6.1: Llama-PLLuM-70B-instruct (only catalogued release, 2412)); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "pllum-12b-instruct-2412",
    "bench": "polish-medical-exams-om",
    "score": 0.359,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'PLLuM-12B-Inst', Avg OM = 35.9% (PLLuM-12B-instruct (only catalogued release, 2412)); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "pllum-12b-chat-2412",
    "bench": "polish-medical-exams-om",
    "score": 0.295,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'PLLuM-12B-Chat', Avg OM = 29.5% (PLLuM-12B-chat (only catalogued release, 2412)); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "polish-medical-exams-om",
    "score": 0.751,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Qwen3.5-122B-A10B', Avg OM = 75.1%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "polish-medical-exams-om",
    "score": 0.722,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Qwen3.5-35B-A3B', Avg OM = 72.2%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "polish-medical-exams-om",
    "score": 0.631,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Qwen3.5-9B', Avg OM = 63.1%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "polish-medical-exams-om",
    "score": 0.658,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'LLaMA3.3-70B', Avg OM = 65.8% (meta-llama/Llama-3.3-70B-Instruct); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "gemma-3-27b",
    "bench": "polish-medical-exams-om",
    "score": 0.592,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Gemma-3-27B', Avg OM = 59.2%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "gemma-3-12b",
    "bench": "polish-medical-exams-om",
    "score": 0.533,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'Gemma-3-12B', Avg OM = 53.3%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "medgemma-4b-it",
    "bench": "polish-medical-exams-om",
    "score": 0.434,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'MedGemma-4B', Avg OM = 43.4%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "biomistral-7b",
    "bench": "polish-medical-exams-om",
    "score": 0.306,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'BioMistral-7B', Avg OM = 30.6%; stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "llama3-openbiollm-70b",
    "bench": "polish-medical-exams-om",
    "score": 0.572,
    "source": "https://arxiv.org/html/2606.12250v1",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Polish Medical Exams (bias-mitigated) (arXiv:2606.12250v1, CC BY-SA 4.0) Table 4 (average performance under the bias-mitigated methodology OM vs the previous protocol PM), row 'OpenBioLLM-70B', Avg OM = 57.2% (aaditya/Llama3-OpenBioLLM-70B); stored /100. Authors' own run: bias-mitigated instruction, constrained generation to A-E, temperature 0, vLLM/Transformers on the authors' A100 cluster; mean over seven exams.",
    "source_id": "paper-2606.12250v1-polish-medical-exams"
  },
  {
    "model": "gpt-5-mini",
    "bench": "viseditbench-pass",
    "score": 0.6,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'GPT-5-Mini', Overall = 60.0%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "gpt-5",
    "bench": "viseditbench-pass",
    "score": 0.6568,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'GPT-5', Overall = 65.68%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "viseditbench-pass",
    "score": 0.6273,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Claude-4.5-Sonnet', Overall = 62.73%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "viseditbench-pass",
    "score": 0.7446,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Claude-4.6-Sonnet', Overall = 74.46%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "gemma-3-4b",
    "bench": "viseditbench-pass",
    "score": 0.2129,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Gemma-3-4B', Overall = 21.29%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "gemma-3-12b",
    "bench": "viseditbench-pass",
    "score": 0.3496,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Gemma-3-12B', Overall = 34.96%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "gemma-3-27b",
    "bench": "viseditbench-pass",
    "score": 0.4295,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Gemma-3-27B', Overall = 42.95%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "pixtral-12b",
    "bench": "viseditbench-pass",
    "score": 0.3532,
    "source": "https://arxiv.org/html/2608.10408v1",
    "by": "independent",
    "date": "2026-08-11",
    "note": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0) Table 2 (pass rate by editing intent), row 'Pixtral-12B', Overall = 35.32%; stored /100. Authors' own run: zero-shot, image detail high, temperature 0.2, top-p 1.0, max 4,096 output tokens; GPT-4o rubric judge (Gemini 2.5 Pro judges GPT-4o's outputs).",
    "source_id": "paper-2608.10408v1-viseditbench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "diagram-mmu-dqa",
    "score": 0.8629,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Gemini-3.1 Pro', DQA Acc. = 86.29%; stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "diagram-mmu-dqa",
    "score": 0.8646,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Gemini-3.0 Pro', DQA Acc. = 86.46% (Gemini 3 Pro (preview API)); stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "diagram-mmu-dqa",
    "score": 0.8407,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Gemini-3.0 Flash', DQA Acc. = 84.07% (Gemini 3 Flash (preview API)); stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "gpt-5-2",
    "bench": "diagram-mmu-dqa",
    "score": 0.8167,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'GPT-5.2', DQA Acc. = 81.67% (reasoning_effort = medium); stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "diagram-mmu-dqa",
    "score": 0.6875,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Claude-4.6 Opus', DQA Acc. = 68.75%; stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "diagram-mmu-dqa",
    "score": 0.8342,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Qwen3.5-397B-A17B', DQA Acc. = 83.42%; stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "kimi-k2-5",
    "bench": "diagram-mmu-dqa",
    "score": 0.7974,
    "source": "https://arxiv.org/html/2608.12262v1",
    "by": "independent",
    "date": "2026-08-12",
    "note": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0) Table 5 (main results on foundational capability), row 'Kimi-K2.5', DQA Acc. = 79.74%; stored /100. Authors' own run (Table D.1): API inference, temperature 1.0, max 16,384 output tokens; direct answering; LLM answer extraction and binary grading.",
    "source_id": "paper-2608.12262v1-diagram-mmu"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "bear-bench-overall",
    "score": 0.754,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3.5-397B-A17B', Overall (n=1000) = 75.4%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "bear-bench-overall",
    "score": 0.751,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Gemini 3.1 Pro', Overall (n=1000) = 75.1%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "bear-bench-overall",
    "score": 0.674,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Gemini 2.5 Pro', Overall (n=1000) = 67.4%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-6-plus",
    "bench": "bear-bench-overall",
    "score": 0.65,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen 3.6 Plus', Overall (n=1000) = 65.0%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "bear-bench-overall",
    "score": 0.627,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Claude Sonnet 4.6', Overall (n=1000) = 62.7%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "bear-bench-overall",
    "score": 0.615,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Claude Opus 4.6', Overall (n=1000) = 61.5%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "gemini-2-5-flash",
    "bench": "bear-bench-overall",
    "score": 0.49,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Gemini 2.5 Flash', Overall (n=1000) = 49.0%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "bear-bench-overall",
    "score": 0.608,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3.5-27B', Overall (n=1000) = 60.8%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bear-bench-overall",
    "score": 0.496,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3.5-9B', Overall (n=1000) = 49.6%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bear-bench-overall",
    "score": 0.486,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3.5-4B', Overall (n=1000) = 48.6%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-vl-4b-thinking",
    "bench": "bear-bench-overall",
    "score": 0.345,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3-VL-4B-Thinking', Overall (n=1000) = 34.5%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "bear-bench-overall",
    "score": 0.254,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3-VL-8B-Instruct', Overall (n=1000) = 25.4%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "gemma-4-31b",
    "bench": "bear-bench-overall",
    "score": 0.235,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'gemma-4-31B-it', Overall (n=1000) = 23.5%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-vl-2b-instruct",
    "bench": "bear-bench-overall",
    "score": 0.143,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3-VL-2B-Instruct', Overall (n=1000) = 14.3%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "qwen3-5-0-8b",
    "bench": "bear-bench-overall",
    "score": 0.106,
    "source": "https://arxiv.org/html/2608.17895v1",
    "by": "independent",
    "date": "2026-08-18",
    "note": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0) Table 2 (BEAR-Bench leaderboard), row 'Qwen3.5-0.8B', Overall (n=1000) = 10.6%; stored /100. Authors' own run: zero-shot, image + raw question, temperature 0.6; open weights on local H100/L40, proprietary via OpenRouter; GPT-4o judge.",
    "source_id": "paper-2608.17895v1-bear-bench"
  },
  {
    "model": "claude-opus-5",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.665,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Claude Opus 5', pass@1 = 66.5%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "gpt-5-4",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.6536,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'GPT-5.4', pass@1 = 65.36%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.6191,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'GPT-5.6-sol', pass@1 = 61.91%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.5919,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Claude Sonnet 4.6', pass@1 = 59.19%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "gpt-6-astra",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.5831,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'GPT-6 Astra', pass@1 = 58.31%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "kimi-k3",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.5737,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Kimi-K3', pass@1 = 57.37%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.517,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Qwen3.8-27B', pass@1 = 51.7% (local vLLM); stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "gpt-5-2",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.4628,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'GPT-5.2', pass@1 = 46.28%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.3209,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Claude Opus 4.6', pass@1 = 32.09%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "kimi-k2-6",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.3766,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Kimi-K2.6', pass@1 = 37.66%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "glm-5-1",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.3319,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'GLM-5.1', pass@1 = 33.19%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.3294,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Qwen3.6-27B', pass@1 = 32.94% (local vLLM); stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "o3-pro",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.1931,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'o3-pro', pass@1 = 19.31%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "grok-4-3",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.1438,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Grok-4.3', pass@1 = 14.38%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.0565,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Qwen3.5-9B', pass@1 = 5.65% (local vLLM); stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "mistral-large-3",
    "bench": "thinkingbox-bench-pass1",
    "score": 0.0466,
    "source": "https://arxiv.org/html/2608.19741v4",
    "by": "independent",
    "date": "2026-10-01",
    "note": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0) Table 15 (repeated-trial discovery and reliability), row 'Mistral-Large-3', pass@1 = 4.66%; stored /100. Authors' own run: same sandbox, prompts, tools and GPT-5.4-mini simulated user for all models; 20 independent trials per task; API models via Azure with temperature 1.0 and reasoning_effort medium (Table 9); Qwen models via local vLLM (Table 10).",
    "source_id": "paper-2608.19741v4-thinkingbox"
  },
  {
    "model": "claude-opus-5",
    "bench": "scidocbench-overall",
    "score": 0.626,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Claude-Opus-5', Overall = 62.6 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "scidocbench-overall",
    "score": 0.5341,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Claude-Opus-4.8', Overall = 53.41 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "scidocbench-overall",
    "score": 0.4957,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Claude-Sonnet-4.6', Overall = 49.57 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "scidocbench-overall",
    "score": 0.61,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'GPT-5.6-Sol', Overall = 61.0 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "scidocbench-overall",
    "score": 0.5419,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'GPT-5.6-Terra', Overall = 54.19 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "scidocbench-overall",
    "score": 0.5058,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'GPT-5.6-Luna', Overall = 50.58 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "scidocbench-overall",
    "score": 0.5987,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Gemini-3.6-Flash', Overall = 59.87 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "scidocbench-overall",
    "score": 0.5588,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Qwen3.7-Plus', Overall = 55.88 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "scidocbench-overall",
    "score": 0.5289,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Qwen3.8-27B', Overall = 52.89 (0-100) (Hugging Face Qwen/Qwen3.8-27B); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "kimi-k2-5",
    "bench": "scidocbench-overall",
    "score": 0.5038,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'Kimi-K2.5', Overall = 50.38 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "glm-4-6v",
    "bench": "scidocbench-overall",
    "score": 0.4244,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'GLM-4.6V', Overall = 42.44 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "mimo-v2-5",
    "bench": "scidocbench-overall",
    "score": 0.4016,
    "source": "https://arxiv.org/html/2609.05141v2",
    "by": "independent",
    "date": "2026-09-28",
    "note": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0) Table 2 (SciDocBench leaderboard), row 'MiMo-V2.5', Overall = 40.16 (0-100); stored /100. Authors' own run: same 496 instances for every model (question + source materials only), failed or unusable responses scored 0; per-question rules / rubric LLM judge / execution scoring.",
    "source_id": "paper-2609.05141v2-scidocbench"
  },
  {
    "model": "claude-opus-5",
    "bench": "gamelogicbench-claude-code",
    "score": 0.5278,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'Claude-Opus-5', Average solve rate = 52.78% (claude-opus-5, released 2026-07-24); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "gamelogicbench-claude-code",
    "score": 0.4444,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'Qwen-3.8-Max', Average solve rate = 44.44% (qwen3.8-max, released 2026-08-03 (undated id)); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "gamelogicbench-claude-code",
    "score": 0.4167,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'GPT-5.6-Sol', Average solve rate = 41.67% (gpt-5.6-sol); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "deepseek-v4-pro",
    "bench": "gamelogicbench-claude-code",
    "score": 0.4167,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'DeepSeek-V4-Pro-0813', Average solve rate = 41.67% (deepseek-v4-pro-0813 snapshot); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "glm-5-2",
    "bench": "gamelogicbench-claude-code",
    "score": 0.375,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'GLM-5.2', Average solve rate = 37.5% (glm-5.2); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "kimi-k3",
    "bench": "gamelogicbench-claude-code",
    "score": 0.3611,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'Kimi-K3', Average solve rate = 36.11% (kimi-k3); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "deepseek-v4-flash",
    "bench": "gamelogicbench-claude-code",
    "score": 0.3194,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'DeepSeek-V4-Flash-0731', Average solve rate = 31.94% (deepseek-v4-flash-0731 snapshot); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "gamelogicbench-claude-code",
    "score": 0.25,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'Claude-Sonnet-5', Average solve rate = 25.0% (claude-sonnet-5); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "gamelogicbench-claude-code",
    "score": 0.25,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'DeepSeek-V4-Pro', Average solve rate = 25.0% (undated = original April 2026 release (Appendix A)); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "minimax-m3",
    "bench": "gamelogicbench-claude-code",
    "score": 0.1944,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'MiniMax-M3', Average solve rate = 19.44% (minimax-m3); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "gamelogicbench-claude-code",
    "score": 0.1667,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'MiMo-V2.5-Pro', Average solve rate = 16.67% (mimo-v2.5-pro); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "gamelogicbench-claude-code",
    "score": 0.1667,
    "source": "https://arxiv.org/html/2609.21562v2",
    "by": "independent",
    "date": "2026-09-21",
    "note": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0) Table 3 (solve rates per tier), Claude Code scaffold block, row 'DeepSeek-V4-Flash', Average solve rate = 16.67% (undated = original April 2026 release (Appendix A)); stored /100. Authors' own run: Claude Code 2.1.177 scaffold, effort=high, 3,600-second limit, one sample per configuration, network egress sealed except the model API; model ids and release dates per Table 6.",
    "source_id": "paper-2609.21562v2-gamelogicbench"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "eegagentbench-overall",
    "score": 0.631,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'GPT-5.6-sol', Overall = 0.631 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "minimax-m3",
    "bench": "eegagentbench-overall",
    "score": 0.53,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'MiniMax-M3', Overall = 0.53 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-7-max",
    "bench": "eegagentbench-overall",
    "score": 0.529,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3.7-Max', Overall = 0.529 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "grok-4-5",
    "bench": "eegagentbench-overall",
    "score": 0.518,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Grok-4.5', Overall = 0.518 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "kimi-k2-6",
    "bench": "eegagentbench-overall",
    "score": 0.513,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Kimi-K2.6', Overall = 0.513 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "eegagentbench-overall",
    "score": 0.501,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3.5-397B-A17B', Overall = 0.501 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "gpt-5-5",
    "bench": "eegagentbench-overall",
    "score": 0.5,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'GPT-5.5', Overall = 0.5 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "eegagentbench-overall",
    "score": 0.495,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Gemini-3.5-Flash', Overall = 0.495 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "hy3",
    "bench": "eegagentbench-overall",
    "score": 0.492,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Hy3', Overall = 0.492 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "mimo-v2-5-pro",
    "bench": "eegagentbench-overall",
    "score": 0.491,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'MiMo-V2.5-Pro', Overall = 0.491 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-5-122b-a10b",
    "bench": "eegagentbench-overall",
    "score": 0.485,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3.5-122B-A10B', Overall = 0.485 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "glm-5-1",
    "bench": "eegagentbench-overall",
    "score": 0.467,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'GLM-5.1', Overall = 0.467 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "eegagentbench-overall",
    "score": 0.463,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Claude-Opus-4.6', Overall = 0.463 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "eegagentbench-overall",
    "score": 0.46,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'DeepSeek-V3.2', Overall = 0.46 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-5-35b-a3b",
    "bench": "eegagentbench-overall",
    "score": 0.459,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3.5-35B-A3B', Overall = 0.459 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "glm-5-2",
    "bench": "eegagentbench-overall",
    "score": 0.455,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'GLM-5.2', Overall = 0.455 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "longcat-2-0",
    "bench": "eegagentbench-overall",
    "score": 0.445,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'LongCat-2', Overall = 0.445 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "eegagentbench-overall",
    "score": 0.442,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Gemini-2.5-Pro', Overall = 0.442 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "gpt-5-4",
    "bench": "eegagentbench-overall",
    "score": 0.438,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'GPT-5.4', Overall = 0.438 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "eegagentbench-overall",
    "score": 0.433,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3.6-27B', Overall = 0.433 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "eegagentbench-overall",
    "score": 0.429,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Llama3.3-70B', Overall = 0.429 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "mistral-medium-3-5",
    "bench": "eegagentbench-overall",
    "score": 0.419,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Mistral-Medium-3.5', Overall = 0.419 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-8b",
    "bench": "eegagentbench-overall",
    "score": 0.332,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3-8B', Overall = 0.332 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-4b",
    "bench": "eegagentbench-overall",
    "score": 0.299,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3-4B', Overall = 0.299 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "eegagentbench-overall",
    "score": 0.233,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'Qwen3-1.7B', Overall = 0.233 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "ernie-4-5-21b-a3b",
    "bench": "eegagentbench-overall",
    "score": 0.228,
    "source": "https://arxiv.org/html/2609.31632v1",
    "by": "independent",
    "date": "2026-09-10",
    "note": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0) Table 2 (main results on EEGAgentBench), row 'ERNIE-4.5-21B-A3B', Overall = 0.228 (0-1); stored as printed. Authors' own run: same 10 EEG tools and interaction limit for every model, single run per instance, each model's own context window.",
    "source_id": "paper-2609.31632v1-eegagentbench"
  },
  {
    "model": "claude-opus-5",
    "bench": "e2e-swe-pass1",
    "score": 0.677,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Claude Opus 5', Pass@1 = 67.7% (effort xhigh); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "claude-fable-5",
    "bench": "e2e-swe-pass1",
    "score": 0.653,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Claude Fable 5', Pass@1 = 65.3% (effort high (xhigh exceeded the 128K output limit)); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "e2e-swe-pass1",
    "score": 0.612,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'GPT-5.6 Sol', Pass@1 = 61.2% (effort max); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "e2e-swe-pass1",
    "score": 0.519,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Claude Opus 4.8', Pass@1 = 51.9% (effort xhigh); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "e2e-swe-pass1",
    "score": 0.518,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'GPT-5.6 Terra', Pass@1 = 51.8% (effort max); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "gpt-5-5",
    "bench": "e2e-swe-pass1",
    "score": 0.461,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'GPT-5.5', Pass@1 = 46.1% (effort xhigh); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "muse-spark-1-3",
    "bench": "e2e-swe-pass1",
    "score": 0.452,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "lab",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Muse Spark 1.3', Pass@1 = 45.2% (effort max; Meta's own model (by: lab)); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "e2e-swe-pass1",
    "score": 0.431,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Claude Sonnet 5', Pass@1 = 43.1% (effort xhigh); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "kimi-k3",
    "bench": "e2e-swe-pass1",
    "score": 0.423,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Kimi K3', Pass@1 = 42.3% (effort high); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "e2e-swe-pass1",
    "score": 0.367,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Gemini 3.8 Flash', Pass@1 = 36.7% (effort high); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "e2e-swe-pass1",
    "score": 0.226,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Claude Opus 4.6', Pass@1 = 22.6% (effort high); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "glm-5-2",
    "bench": "e2e-swe-pass1",
    "score": 0.222,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'GLM-5.2', Pass@1 = 22.2% (effort high); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "gemini-3-6-flash",
    "bench": "e2e-swe-pass1",
    "score": 0.117,
    "source": "https://arxiv.org/html/2609.38335v1",
    "by": "independent",
    "date": "2026-09-29",
    "note": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0) Table 2 (evaluation results on the 186-task E2E-SWE corpus), row 'Gemini 3.6 Flash', Pass@1 = 11.7% (effort high); stored /100. Authors' own run: shared ReAct agent scaffold (mini-SWE-agent-like with edit and submit tools), 4-hour wall-clock budget per rollout, 4 independent runs averaged, reasoning effort as printed in the row.",
    "source_id": "paper-2609.38335v1-e2e-swe"
  },
  {
    "model": "claude-opus-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.294,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Opus 5', Strict pass@1 = 29.4%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-opus-5-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.288,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Opus 5.5', Strict pass@1 = 28.8%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "klinikebench-strict-pass1",
    "score": 0.168,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Opus 4.8', Strict pass@1 = 16.8%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.138,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Sonnet 5', Strict pass@1 = 13.8%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.099,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Sonnet 4.5', Strict pass@1 = 9.9%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "klinikebench-strict-pass1",
    "score": 0.084,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Sonnet 4.6', Strict pass@1 = 8.4%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.027,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Claude Haiku 4.5', Strict pass@1 = 2.7%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-6-astra",
    "bench": "klinikebench-strict-pass1",
    "score": 0.243,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-6-astra', Strict pass@1 = 24.3%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "klinikebench-strict-pass1",
    "score": 0.237,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.6-sol', Strict pass@1 = 23.7%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-4",
    "bench": "klinikebench-strict-pass1",
    "score": 0.234,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.4', Strict pass@1 = 23.4%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-5",
    "bench": "klinikebench-strict-pass1",
    "score": 0.168,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.5', Strict pass@1 = 16.8%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "klinikebench-strict-pass1",
    "score": 0.159,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.4-mini', Strict pass@1 = 15.9%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-1",
    "bench": "klinikebench-strict-pass1",
    "score": 0.153,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.1', Strict pass@1 = 15.3%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-6-terra",
    "bench": "klinikebench-strict-pass1",
    "score": 0.084,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.6-terra', Strict pass@1 = 8.4%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "klinikebench-strict-pass1",
    "score": 0.075,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.6-luna', Strict pass@1 = 7.5%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-4-1",
    "bench": "klinikebench-strict-pass1",
    "score": 0.075,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-4.1', Strict pass@1 = 7.5% (only snapshot gpt-4.1-2025-04-14); stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "klinikebench-strict-pass1",
    "score": 0.036,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-5.3-codex', Strict pass@1 = 3.6%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "klinikebench-strict-pass1",
    "score": 0.021,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-4.1-mini', Strict pass@1 = 2.1% (only snapshot gpt-4.1-mini-2025-04-14); stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "klinikebench-strict-pass1",
    "score": 0.009,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-4.1-nano', Strict pass@1 = 0.9% (only snapshot gpt-4.1-nano-2025-04-14); stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "klinikebench-strict-pass1",
    "score": 0.006,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GPT-4o-mini', Strict pass@1 = 0.6% (only snapshot gpt-4o-mini-2024-07-18); stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "klinikebench-strict-pass1",
    "score": 0.189,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Gemini-3.8-flash', Strict pass@1 = 18.9%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "klinikebench-strict-pass1",
    "score": 0.156,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Gemini-3.5-flash', Strict pass@1 = 15.6%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "gemini-3-7-flash",
    "bench": "klinikebench-strict-pass1",
    "score": 0.108,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Gemini-3.7-flash', Strict pass@1 = 10.8%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "grok-4-7",
    "bench": "klinikebench-strict-pass1",
    "score": 0.222,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Grok-4.7', Strict pass@1 = 22.2%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "grok-4-6",
    "bench": "klinikebench-strict-pass1",
    "score": 0.219,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Grok-4.6', Strict pass@1 = 21.9%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "grok-4-3",
    "bench": "klinikebench-strict-pass1",
    "score": 0.213,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Grok-4.3', Strict pass@1 = 21.3%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "kimi-k2-6",
    "bench": "klinikebench-strict-pass1",
    "score": 0.234,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'Kimi-K2.6', Strict pass@1 = 23.4%; stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "glm-5-1",
    "bench": "klinikebench-strict-pass1",
    "score": 0.048,
    "source": "https://arxiv.org/html/2609.38480v2",
    "by": "independent",
    "date": "2026-10-06",
    "note": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0) Table 7 (results for 31 models on 333 tasks), row 'GLM-5.1', Strict pass@1 = 4.8% (Diagnosis 24.9%, Must-ask 30.8% in the same row); stored /100. Authors' own run: Harbor + Terminus-2 harness, one rollout per task, medium reasoning effort for all models, 32-turn limit, GPT-5.4-mini as patient simulator and must-ask judge.",
    "source_id": "paper-2609.38480v2-klinikebench"
  },
  {
    "model": "claude-opus-4-7",
    "bench": "longsocialbench-full",
    "score": 0.56,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Claude-Opus-4.7', Full (full-context micro accuracy) = 56.0%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "longsocialbench-full",
    "score": 0.515,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Gemini-2.5-Pro', Full (full-context micro accuracy) = 51.5%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "gpt-5",
    "bench": "longsocialbench-full",
    "score": 0.529,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'GPT-5', Full (full-context micro accuracy) = 52.9%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "longsocialbench-full",
    "score": 0.514,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Claude-Sonnet-4.5', Full (full-context micro accuracy) = 51.4%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "longsocialbench-full",
    "score": 0.52,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Claude-4', Full (full-context micro accuracy) = 52.0% (Sec. 4 lists 'Claude Sonnet 4.6, 4.5, and 4', so Claude-4 = Claude Sonnet 4); stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "longsocialbench-full",
    "score": 0.443,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Claude-Sonnet-4.6', Full (full-context micro accuracy) = 44.3%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "glm-5",
    "bench": "longsocialbench-full",
    "score": 0.415,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'GLM-5', Full (full-context micro accuracy) = 41.5%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "deepseek-r1",
    "bench": "longsocialbench-full",
    "score": 0.473,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'DeepSeek-R1', Full (full-context micro accuracy) = 47.3% (Hugging Face DeepSeek-R1 (original January 2025 release)); stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "mistral-large-3",
    "bench": "longsocialbench-full",
    "score": 0.42,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Mistral-Large-3', Full (full-context micro accuracy) = 42.0%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "llama-4-maverick",
    "bench": "longsocialbench-full",
    "score": 0.412,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Llama-4-Mav-17B', Full (full-context micro accuracy) = 41.2%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "llama-4-scout",
    "bench": "longsocialbench-full",
    "score": 0.402,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Llama-4-Scout-17B', Full (full-context micro accuracy) = 40.2%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "longsocialbench-full",
    "score": 0.38,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Llama-3.1-70B', Full (full-context micro accuracy) = 38.0% (Hugging Face chat model; Instruct assumed); stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "qwen3-32b",
    "bench": "longsocialbench-full",
    "score": 0.419,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Qwen3-32B', Full (full-context micro accuracy) = 41.9%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "longsocialbench-full",
    "score": 0.395,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'GPT-OSS-120B', Full (full-context micro accuracy) = 39.5%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "longsocialbench-full",
    "score": 0.363,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'GPT-OSS-20B', Full (full-context micro accuracy) = 36.3%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "llama-3-2-90b-vision",
    "bench": "longsocialbench-full",
    "score": 0.356,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Llama-3.2-90B', Full (full-context micro accuracy) = 35.6% (Hugging Face chat model; Vision Instruct assumed); stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "gemma-3-27b",
    "bench": "longsocialbench-full",
    "score": 0.365,
    "source": "https://arxiv.org/html/2610.04118v1",
    "by": "independent",
    "date": "2026-10-02",
    "note": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0) Table 13 (per-model accuracy on three representative workflows), row 'Gemma-3-27B', Full (full-context micro accuracy) = 36.5%; stored /100. Authors' own run: full-context prompt with the whole serialized thread, four options, one answer letter; exact match over parsed responses; context cap per Appendix Table 11.",
    "source_id": "paper-2610.04118v1-longsocialbench"
  },
  {
    "model": "claude-opus-5",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.4156,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'Claude Opus5', V2WScore (0-100) = 41.56 (Table A7: anthropic/claude-opus-5 via Anthropic); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "kimi-k3",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.3319,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'Kimi K3', V2WScore (0-100) = 33.19 (Table A7: moonshotai/kimi-k3, provider moonshotai/mxfp4); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "gpt-5-6-sol",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.31,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'GPT-5.6 Sol', V2WScore (0-100) = 31.0 (Table A7: openai/gpt-5.6-sol via OpenAI); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "deepseek-v4-1-flash",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.3054,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'DeepSeek-V4.1 Flash', V2WScore (0-100) = 30.54 (Table A7: deepseek/deepseek-v4.1-flash, provider novita/fp8); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "gemini-3-8-flash",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.3011,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'Gemini-3.8 Flash', V2WScore (0-100) = 30.11 (Table A7: google/gemini-3.8-flash via Google Vertex, effort High); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "qwen3-8-max-0902",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.2655,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'Qwen-3.8 Max-0902', V2WScore (0-100) = 26.55 (Table A7: qwen/qwen3.8-max-0902 via Alibaba); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "glm-5-3-flash",
    "bench": "video2world-v2wscore-opencode",
    "score": 0.2569,
    "source": "https://arxiv.org/html/2610.04432v2",
    "by": "independent",
    "date": "2026-10-07",
    "note": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0) Table 2 (overall performance on Video2World), row 'GLM-5.3 Flash', V2WScore (0-100) = 25.69 (Table A7: z-ai/glm-5.3-flash, provider novita/fp8); stored /100. Authors' own run: OpenCode 1.18.29 scaffold, one autonomous attempt per instance, 180-minute wall-clock limit, model identifier and provider route per Table A7 (reasoning effort unspecified unless noted).",
    "source_id": "paper-2610.04432v2-video2world"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "spacecast-bench-overall",
    "score": 0.36,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Claude Sonnet 4.6', Overall = 36.0%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "claude-sonnet-5",
    "bench": "spacecast-bench-overall",
    "score": 0.383,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Claude Sonnet 5', Overall = 38.3%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "spacecast-bench-overall",
    "score": 0.58,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Gemini 3.5 Flash', Overall = 58.0%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "spacecast-bench-overall",
    "score": 0.567,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Gemini 3 Flash', Overall = 56.7%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "gpt-5-4",
    "bench": "spacecast-bench-overall",
    "score": 0.377,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'GPT-5.4', Overall = 37.7%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "gpt-5-6-luna",
    "bench": "spacecast-bench-overall",
    "score": 0.466,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'GPT-5.6-Luna', Overall = 46.6%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-6-flash",
    "bench": "spacecast-bench-overall",
    "score": 0.477,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.6-Flash', Overall = 47.7%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-7-plus",
    "bench": "spacecast-bench-overall",
    "score": 0.553,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.7-Plus', Overall = 55.3%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-5-27b",
    "bench": "spacecast-bench-overall",
    "score": 0.393,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.5-27B', Overall = 39.3%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "spacecast-bench-overall",
    "score": 0.346,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.5-9B', Overall = 34.6%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "spacecast-bench-overall",
    "score": 0.337,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.5-4B', Overall = 33.7%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "qwen3-5-2b",
    "bench": "spacecast-bench-overall",
    "score": 0.313,
    "source": "https://arxiv.org/html/2610.12402v1",
    "by": "independent",
    "date": "2026-10-08",
    "note": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0) Table 1 (accuracy on all 16 SpaceCast-Bench task types), row 'Qwen3.5-2B', Overall = 31.3%; stored /100. Authors' own run: chain-of-thought prompt, deterministic decoding, macro-average accuracy over 16 task types.",
    "source_id": "paper-2610.12402v1-spacecast-bench"
  },
  {
    "model": "bloom-1b7",
    "bench": "open-pt-llm-avg9",
    "score": 0.2856,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'bigscience/bloom-1b7' (source=hub, precision=float16, type=pretrained); published average 28.56 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha cc72a88036. Identity: catalogue bloom-1b7 (HF bigscience/bloom-1b7).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bloom-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3354,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'bigscience/bloom-3b' (source=hub, precision=float16, type=pretrained); published average 33.54 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 52bc5b4301. Identity: catalogue bloom-3b (HF bigscience/bloom-3b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bloom-560m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2543,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'bigscience/bloom-560m' (source=hub, precision=float16, type=pretrained); published average 25.43 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha ac2ae5fab2. Identity: catalogue bloom-560m (HF bigscience/bloom-560m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bloom-7b1",
    "bench": "open-pt-llm-avg9",
    "score": 0.2887,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'bigscience/bloom-7b1' (source=hub, precision=float16, type=pretrained); published average 28.87 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 6232703e39. Identity: catalogue bloom-7b1 (HF bigscience/bloom-7b1).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "deepseek-r1-distill-llama-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4239,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-R1-Distill-Llama-8B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 42.39 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 24ae87a9c3. Identity: catalogue deepseek-r1-distill-llama-8b (HF deepseek-ai/DeepSeek-R1-Distill-Llama-8B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2534,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-01-28",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 25.34 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 0092dee1c3. Identity: catalogue deepseek-r1-distill-qwen-1-5b (HF deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "deepseek-r1-distill-qwen-14b",
    "bench": "open-pt-llm-avg9",
    "score": 0.681,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-R1-Distill-Qwen-14B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 68.1 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha . Identity: catalogue deepseek-r1-distill-qwen-14b (HF deepseek-ai/DeepSeek-R1-Distill-Qwen-14B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.504,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-01-28",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 50.4 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 09db661939. Identity: catalogue deepseek-r1-distill-qwen-7b (HF deepseek-ai/DeepSeek-R1-Distill-Qwen-7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "falcon-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3686,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'tiiuae/falcon-7b' (source=hub, precision=bfloat16, type=pretrained); published average 36.86 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 898df1396f. Identity: catalogue falcon-7b (HF tiiuae/falcon-7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemma-2b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.4407,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-2b' (source=hub, precision=bfloat16, type=pretrained); published average 44.07 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 9d067f00de. Identity: catalogue gemma-2b-base (HF google/gemma-2b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemma-7b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.6418,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-21",
    "note": "Open PT-LLM Leaderboard row 'google/gemma-7b' (source=hub, precision=bfloat16, type=pretrained); published average 64.18 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 227defe192. Identity: catalogue gemma-7b-base (HF google/gemma-7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-2-large",
    "bench": "open-pt-llm-avg9",
    "score": 0.2739,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'openai-community/gpt2-large' (source=hub, precision=float16, type=pretrained); published average 27.39 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 97935fc1a4. Identity: catalogue gpt-2-large (HF openai-community/gpt2-large).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-2-medium",
    "bench": "open-pt-llm-avg9",
    "score": 0.2906,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'openai-community/gpt2-medium' (source=hub, precision=float16, type=pretrained); published average 29.06 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f65d4965d1. Identity: catalogue gpt-2-medium (HF openai-community/gpt2-medium).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-2-xl",
    "bench": "open-pt-llm-avg9",
    "score": 0.2586,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'openai-community/gpt2-xl' (source=hub, precision=float16, type=pretrained); published average 25.86 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha b81cf803a4. Identity: catalogue gpt-2-xl (HF openai-community/gpt2-xl).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-j-6b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3571,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/gpt-j-6b' (source=hub, precision=float16, type=pretrained); published average 35.71 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 47e169305d. Identity: catalogue gpt-j-6b (HF EleutherAI/gpt-j-6b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2668,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/gpt-neo-1.3B' (source=hub, precision=float16, type=pretrained); published average 26.68 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha dbe59a7f4a. Identity: catalogue gpt-neo-1-3b (HF EleutherAI/gpt-neo-1.3B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-neo-125m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2555,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/gpt-neo-125m' (source=hub, precision=float16, type=pretrained); published average 25.55 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 21def0189f. Identity: catalogue gpt-neo-125m (HF EleutherAI/gpt-neo-125m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2804,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/gpt-neo-2.7B' (source=hub, precision=float16, type=pretrained); published average 28.04 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha e24fa29113. Identity: catalogue gpt-neo-2-7b (HF EleutherAI/gpt-neo-2.7B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-neox-20b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2832,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/gpt-neox-20b' (source=hub, precision=float16, type=pretrained); published average 28.32 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha c292233c83. Identity: catalogue gpt-neox-20b (HF EleutherAI/gpt-neox-20b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-oss-120b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4296,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-29",
    "note": "Open PT-LLM Leaderboard row 'openai/gpt-oss-120b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 42.96 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha b5c939de8f. Identity: catalogue gpt-oss-120b (HF openai/gpt-oss-120b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "open-pt-llm-avg9",
    "score": 0.4283,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-29",
    "note": "Open PT-LLM Leaderboard row 'openai/gpt-oss-20b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 42.83 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 6cee5e81ee. Identity: catalogue gpt-oss-20b (HF openai/gpt-oss-20b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "lfm2-1-2b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5869,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'LiquidAI/LFM2-1.2B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 58.69 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 5d7e47e946. Identity: catalogue lfm2-1-2b (HF LiquidAI/LFM2-1.2B).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "lfm2-350m",
    "bench": "open-pt-llm-avg9",
    "score": 0.3216,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'LiquidAI/LFM2-350M' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 32.16 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 8707ee0cbe. Identity: catalogue lfm2-350m (HF LiquidAI/LFM2-350M).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "lfm2-700m",
    "bench": "open-pt-llm-avg9",
    "score": 0.5503,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'LiquidAI/LFM2-700M' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 55.03 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha ab26029373. Identity: catalogue lfm2-700m (HF LiquidAI/LFM2-700M).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-2-13b",
    "bench": "open-pt-llm-avg9",
    "score": 0.5749,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-13b-hf' (source=hub, precision=float16, type=pretrained); published average 57.49 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha dc1d3b3bfd. Identity: catalogue llama-2-13b (HF meta-llama/Llama-2-13b-hf).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-2-70b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6773,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-70b-hf' (source=hub, precision=float16, type=pretrained); published average 67.73 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 90052941a6. Identity: catalogue llama-2-70b (HF meta-llama/Llama-2-70b-hf).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.489,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Llama-2-7b-hf' (source=hub, precision=float16, type=pretrained); published average 48.9 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 8cca527612. Identity: catalogue llama-2-7b (HF meta-llama/Llama-2-7b-hf).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.6113,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mistral-7B-v0.1' (source=hub, precision=bfloat16, type=pretrained); published average 61.13 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 26bca36bde. Identity: catalogue mistral-7b-v0-1 (HF mistralai/Mistral-7B-v0.1).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "mixtral-8x7b-v0-1",
    "bench": "open-pt-llm-avg9",
    "score": 0.6787,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'mistralai/Mixtral-8x7B-v0.1' (source=hub, precision=bfloat16, type=pretrained); published average 67.87 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 985aa05589. Identity: catalogue mixtral-8x7b-v0-1 (HF mistralai/Mixtral-8x7B-v0.1).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-1-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2555,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-1.3b' (source=hub, precision=float16, type=pretrained); published average 25.55 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 3f5c25d0bc. Identity: catalogue opt-1-3b (HF facebook/opt-1.3b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-125m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2437,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-125m' (source=hub, precision=float16, type=pretrained); published average 24.37 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 27dcfa74d3. Identity: catalogue opt-125m (HF facebook/opt-125m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-13b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2822,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-13b' (source=hub, precision=float16, type=pretrained); published average 28.22 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha e515202d1e. Identity: catalogue opt-13b (HF facebook/opt-13b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.27,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-2.7b' (source=hub, precision=float16, type=pretrained); published average 27.0 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 905a4b602c. Identity: catalogue opt-2-7b (HF facebook/opt-2.7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-30b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3954,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-30b' (source=hub, precision=float16, type=pretrained); published average 39.54 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha ceea0a90ac. Identity: catalogue opt-30b (HF facebook/opt-30b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-350m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2332,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-350m' (source=hub, precision=float16, type=pretrained); published average 23.32 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 08ab08cc4b. Identity: catalogue opt-350m (HF facebook/opt-350m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-6-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2865,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-6.7b' (source=hub, precision=float16, type=pretrained); published average 28.65 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a45aa65bbe. Identity: catalogue opt-6-7b (HF facebook/opt-6.7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "opt-66b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3499,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'facebook/opt-66b' (source=hub, precision=float16, type=pretrained); published average 34.99 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 7259969061. Identity: catalogue opt-66b (HF facebook/opt-66b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "phi-1-5",
    "bench": "open-pt-llm-avg9",
    "score": 0.2964,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'microsoft/phi-1_5' (source=hub, precision=float16, type=pretrained); published average 29.64 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha bffd3b29c4. Identity: catalogue phi-1-5 (HF microsoft/phi-1_5).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "phi-2",
    "bench": "open-pt-llm-avg9",
    "score": 0.3652,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'microsoft/phi-2' (source=hub, precision=float16, type=pretrained); published average 36.52 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha b10c3eba54. Identity: catalogue phi-2 (HF microsoft/phi-2).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "open-pt-llm-avg9",
    "score": 0.6792,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-4-reasoning' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 67.92 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a2b4ce1926. Identity: catalogue phi-4-reasoning (HF microsoft/Phi-4-reasoning).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "open-pt-llm-avg9",
    "score": 0.6595,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-05-01",
    "note": "Open PT-LLM Leaderboard row 'microsoft/Phi-4-reasoning-plus' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 65.95 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 459ab9fc2d. Identity: catalogue phi-4-reasoning-plus (HF microsoft/Phi-4-reasoning-plus).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-12b",
    "bench": "open-pt-llm-avg9",
    "score": 0.3401,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-12b' (source=hub, precision=float16, type=pretrained); published average 34.01 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 35c9d7f32f. Identity: catalogue pythia-12b (HF EleutherAI/pythia-12b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-160m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2945,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-160m' (source=hub, precision=float16, type=pretrained); published average 29.45 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 50f5173d93. Identity: catalogue pythia-160m (HF EleutherAI/pythia-160m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-1b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2523,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-1b' (source=hub, precision=float16, type=pretrained); published average 25.23 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f73d7dcc54. Identity: catalogue pythia-1b (HF EleutherAI/pythia-1b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-2-8b",
    "bench": "open-pt-llm-avg9",
    "score": 0.24,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-2.8b' (source=hub, precision=float16, type=pretrained); published average 24.0 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 2a259cdd96. Identity: catalogue pythia-2-8b (HF EleutherAI/pythia-2.8b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-410m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2637,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-410m' (source=hub, precision=float16, type=pretrained); published average 26.37 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 9879c9b5f8. Identity: catalogue pythia-410m (HF EleutherAI/pythia-410m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-6-9b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2904,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-6.9b' (source=hub, precision=float16, type=pretrained); published average 29.04 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha f271943e88. Identity: catalogue pythia-6-9b (HF EleutherAI/pythia-6.9b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "pythia-70m",
    "bench": "open-pt-llm-avg9",
    "score": 0.2318,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Open PT-LLM Leaderboard row 'EleutherAI/pythia-70m' (source=hub, precision=float16, type=pretrained); published average 23.18 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha a39f36b100. Identity: catalogue pythia-70m (HF EleutherAI/pythia-70m).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "qwen3-4b-thinking-2507",
    "bench": "open-pt-llm-avg9",
    "score": 0.5907,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-08-30",
    "note": "Open PT-LLM Leaderboard row 'Qwen/Qwen3-4B-Thinking-2507' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 59.07 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 768f209d9e. Identity: catalogue qwen3-4b-thinking-2507 (HF Qwen/Qwen3-4B-Thinking-2507).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "redpajama-incite-7b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.3297,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'togethercomputer/RedPajama-INCITE-7B-Base' (source=hub, precision=float16, type=pretrained); published average 32.97 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 78f7e48244. Identity: catalogue redpajama-incite-7b-base (HF togethercomputer/RedPajama-INCITE-7B-Base).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "redpajama-incite-base-3b-v1",
    "bench": "open-pt-llm-avg9",
    "score": 0.297,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'togethercomputer/RedPajama-INCITE-Base-3B-v1' (source=hub, precision=float16, type=pretrained); published average 29.7 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 094fbdd0c9. Identity: catalogue redpajama-incite-base-3b-v1 (HF togethercomputer/RedPajama-INCITE-Base-3B-v1).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-3b-4e1t",
    "bench": "open-pt-llm-avg9",
    "score": 0.3979,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-3b-4e1t' (source=hub, precision=bfloat16, type=pretrained); published average 39.79 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha e3be657f4d. Identity: catalogue stablelm-3b-4e1t (HF stabilityai/stablelm-3b-4e1t).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-base-alpha-3b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2764,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-base-alpha-3b' (source=hub, precision=float16, type=pretrained); published average 27.64 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 819657db42. Identity: catalogue stablelm-base-alpha-3b (HF stabilityai/stablelm-base-alpha-3b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "stablelm-base-alpha-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2545,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'stabilityai/stablelm-base-alpha-7b' (source=hub, precision=float16, type=pretrained); published average 25.45 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 1d43a3b40d. Identity: catalogue stablelm-base-alpha-7b (HF stabilityai/stablelm-base-alpha-7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "tulu-2-13b",
    "bench": "open-pt-llm-avg9",
    "score": 0.635,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-22",
    "note": "Open PT-LLM Leaderboard row 'allenai/tulu-2-13b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 63.5 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 9c8375d9f2. Identity: catalogue tulu-2-13b (HF allenai/tulu-2-13b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "tulu-2-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.63,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-05-22",
    "note": "Open PT-LLM Leaderboard row 'allenai/tulu-2-7b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 63.0 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 3c6e328ae9. Identity: catalogue tulu-2-7b (HF allenai/tulu-2-7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "tulu-2-dpo-13b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6577,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-03-05",
    "note": "Open PT-LLM Leaderboard row 'allenai/tulu-2-dpo-13b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 65.77 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha aa3e5cfac9. Identity: catalogue tulu-2-dpo-13b (HF allenai/tulu-2-dpo-13b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "tulu-2-dpo-7b",
    "bench": "open-pt-llm-avg9",
    "score": 0.6395,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-03-05",
    "note": "Open PT-LLM Leaderboard row 'allenai/tulu-2-dpo-7b' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 63.95 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 8ffe1eb804. Identity: catalogue tulu-2-dpo-7b (HF allenai/tulu-2-dpo-7b).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "open-pt-llm-avg9",
    "score": 0.2615,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-01-29",
    "note": "Open PT-LLM Leaderboard row 'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B' (source=hub, precision=bfloat16, type=chat (RLHF, DPO, IFT, ...)); published average 26.15 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha d66bcfc2f3. Identity: catalogue deepseek-r1-distill-qwen-32b (HF deepseek-ai/DeepSeek-R1-Distill-Qwen-32B).",
    "invalid": "Failed run: several task scores fall far below the trivial baselines (OAB Exams 5.06 on 4-option questions, PT Hate Speech 3.73 and tweetSentBR 3.42 macro-F1) while ASSIN2 RTE is 89.84, i.e. most answers in those tasks were not parsed (reasoning output); the 26.15 average (random baseline 32.64) is not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "command-r-03-2024",
    "bench": "open-pt-llm-avg9",
    "score": 0.6828,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-04-05",
    "note": "Open PT-LLM Leaderboard row 'CohereForAI/c4ai-command-r-v01' (source=hub, precision=float16, type=chat (RLHF, DPO, IFT, ...)); published average 68.28 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 16881ccde1. Identity: catalogue command-r-03-2024 (HF CohereLabs/c4ai-command-r-v01). Row path CohereForAI/c4ai-command-r-v01 is the same Hugging Face repository as the catalogue's CohereLabs/c4ai-command-r-v01 (Cohere renamed the CohereForAI organisation to CohereLabs; the old path redirects).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gpt-2-small",
    "bench": "open-pt-llm-avg9",
    "score": 0.2331,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 'gpt2' (source=hub, precision=float16, type=pretrained); published average 23.31 (divided by 100). Local harness run, float16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 11c5a3d581. Identity: catalogue gpt-2-small (HF openai-community/gpt2). Row path 'gpt2' is the legacy root-namespace name of openai-community/gpt2 (the catalogue identity note: 'formerly the root-namespace gpt2 repo').",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "t5-small",
    "bench": "open-pt-llm-avg9",
    "score": 0.1263,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 't5-small' (source=hub, precision=bfloat16, type=pretrained); published average 12.63 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha df1b051c49. Identity: catalogue t5-small (HF google-t5/t5-small). Row path 't5-small' is the legacy root-namespace name of google-t5/t5-small (same repository).",
    "invalid": "Failed run: average 12.63 vs the leaderboard's random baseline 32.64; the original T5 encoder-decoder checkpoint does not follow the few-shot generative answer format (every task near or below chance), so this is not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "t5-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.1368,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 't5-base' (source=hub, precision=bfloat16, type=pretrained); published average 13.68 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha fe6d9bf207. Identity: catalogue t5-base (HF google-t5/t5-base). Row path 't5-base' is the legacy root-namespace name of google-t5/t5-base (same repository).",
    "invalid": "Failed run: average 13.68 vs the leaderboard's random baseline 32.64; the original T5 encoder-decoder checkpoint does not follow the few-shot generative answer format (every task near or below chance), so this is not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "t5-large",
    "bench": "open-pt-llm-avg9",
    "score": 0.1476,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-02-05",
    "note": "Open PT-LLM Leaderboard row 't5-large' (source=hub, precision=bfloat16, type=pretrained); published average 14.76 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 150ebc2c4b. Identity: catalogue t5-large (HF google-t5/t5-large). Row path 't5-large' is the legacy root-namespace name of google-t5/t5-large (same repository).",
    "invalid": "Failed run: average 14.76 vs the leaderboard's random baseline 32.64; the original T5 encoder-decoder checkpoint does not follow the few-shot generative answer format (every task near or below chance), so this is not a measurement of ability.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "llama-3-1-8b-base",
    "bench": "open-pt-llm-avg9",
    "score": 0.6536,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-07-23",
    "note": "Open PT-LLM Leaderboard row 'meta-llama/Meta-Llama-3.1-8B' (source=hub, precision=bfloat16, type=pretrained); published average 65.36 (divided by 100). Local harness run, bfloat16. Date is the leaderboard's row date (evaluation/submission), not model release. Evaluated revision main = model_sha 13f04ed6f8. Identity: catalogue llama-3-1-8b-base (HF meta-llama/Llama-3.1-8B). Row path meta-llama/Meta-Llama-3.1-8B is the same Hugging Face repository as the catalogue's meta-llama/Llama-3.1-8B (Meta renamed the Llama 3.1 repos; the old path redirects).",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "open-pt-llm-avg9",
    "score": 0.8056,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2025-04-03",
    "note": "Open PT-LLM Leaderboard row 'Gemini 2.0 Flash Lite (001) [2025-04-03]' (source=external, precision=?); published average 80.56 (divided by 100). API run (external); versioned gemini-2.0-flash-lite-001. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "bielik-1-5b-v3-0-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7178,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '62.36 ± 1.17 / 71.78 ± 0.94' (MCC / accuracy); entry 'speakleash/Bielik-1.5B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-1-5b-v3-0-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.5065,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '35.25 ± 1.58 / 50.65 ± 1.31' (MCC / accuracy); entry 'speakleash/Bielik-1.5B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8383,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '78.29 ± 0.81 / 83.83 ± 0.58' (MCC / accuracy); entry 'speakleash/Bielik-11B-v2.3-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-11b-v2-6-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8016,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '72.93 ± 1.20 / 80.16 ± 0.87' (MCC / accuracy); entry 'speakleash/Bielik-11B-v2.6-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-11b-v2-6-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7617,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '68.51 ± 1.72 / 76.17 ± 1.34' (MCC / accuracy); entry 'speakleash/Bielik-11B-v2.6-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-11b-v3-0-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8266,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '76.63 ± 1.53 / 82.66 ± 1.13' (MCC / accuracy); entry 'speakleash/Bielik-11B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-11b-v3-0-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7743,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '70.19 ± 2.43 / 77.43 ± 1.95' (MCC / accuracy); entry 'speakleash/Bielik-11B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.10.0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-4-5b-v3-0-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8049,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '73.42 ± 1.11 / 80.49 ± 0.85' (MCC / accuracy); entry 'speakleash/Bielik-4.5B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-4-5b-v3-0-instruct",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7219,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '63.74 ± 1.33 / 72.19 ± 1.18' (MCC / accuracy); entry 'speakleash/Bielik-4.5B-v3.0-Instruct', instruction-tuned (📝), default setting; EuroEval version 16.11.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "opencoder-8b-instruct",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.4623,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '24.64 ± 2.14 / 46.23 ± 1.58' (MCC / accuracy); entry 'infly/OpenCoder-8B-Instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5281,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '41.25 ± 11.28 / 52.81 ± 8.75' (MCC / accuracy); entry 'microsoft/Phi-4-reasoning-plus', reasoning (🤔), reasoning model at EuroEval default settings; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8234,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '76.19 ± 1.82 / 82.34 ± 1.41' (MCC / accuracy); entry 'meta-llama/Meta-Llama-3-8B', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-3-70b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.9182,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '88.89 ± 1.08 / 91.82 ± 0.80' (MCC / accuracy); entry 'meta-llama/Meta-Llama-3-70B', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-3-1-8b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8355,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '77.82 ± 1.54 / 83.55 ± 1.15' (MCC / accuracy); entry 'meta-llama/Llama-3.1-8B', base model (🧠), default setting; EuroEval version 16.5.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-3-1-8b-base",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.4603,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '32.23 ± 2.11 / 46.03 ± 1.65' (MCC / accuracy); entry 'meta-llama/Llama-3.1-8B', base model (🧠), default setting; EuroEval version 16.5.0.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7627,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '68.96 ± 2.19 / 76.27 ± 1.93' (MCC / accuracy); entry 'mistralai/Mistral-7B-v0.1', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "yi-1-5-34b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.883,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '84.01 ± 0.85 / 88.30 ± 0.62' (MCC / accuracy); entry '01-ai/Yi-1.5-34B', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "yi-1-5-34b-base",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.9589,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '94.53 ± 0.20 / 95.89 ± 0.15' (MCC / accuracy); entry '01-ai/Yi-1.5-34B', base model (🧠), default setting; EuroEval version 15.8.2.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.8303,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '76.72 ± 1.56 / 83.03 ± 1.12' (MCC / accuracy); entry '01-ai/Yi-1.5-9B', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.7941,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '73.12 ± 0.87 / 79.41 ± 0.68' (MCC / accuracy); entry '01-ai/Yi-1.5-9B', base model (🧠), default setting; EuroEval version 15.8.2.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "yi-1-5-6b-base",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7736,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '69.09 ± 1.55 / 77.36 ± 1.16' (MCC / accuracy); entry '01-ai/Yi-1.5-6B', base model (🧠), default setting; EuroEval version 15.10.1.dev0.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "phi-3-mini-4k-instruct-june-2024",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.802,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; cell '72.81 ± 0.93 / 80.20 ± 0.69' (MCC / accuracy); entry 'microsoft/Phi-3-mini-4k-instruct', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0. Checkpoint: the entry names the repo without a revision and the cell was produced with EuroEval 15.10.1.dev0 (development version after the 15.10.0 release on PyPI, 2025-06-17, before 15.10.1 on 2025-06-20), when the repo's main revision held Microsoft's June 2024 update (pushed July 2024), so catalogue phi-3-mini-4k-instruct-june-2024, not the April 2024 weights.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "bielik-7b-instruct-v0-1",
    "bench": "polish-eq-bench-v2",
    "score": 0.3427,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2024-06-18",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik_v0.1', model path speakleash/Bielik-7B-Instruct-v0.1; eq-bench_v2_pl score 34.27 (divided by 100); 156.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-18 14:01:46. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 34.17 (2024-06-18). Identity: catalogue bielik-7b-instruct-v0-1 (HF speakleash/Bielik-7B-Instruct-v0.1). SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab (as for the Bielik CPTUB rows). Run after the public repo was created (2024-03-30).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v2-3-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.7086,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2024-09-14",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik_v2.3', model path speakleash/Bielik-11B-v2.3-Instruct; eq-bench_v2_pl score 70.86 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-09-14 10:40:57. Identity: catalogue bielik-11b-v2-3-instruct (HF speakleash/Bielik-11B-v2.3-Instruct). SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab (as for the Bielik CPTUB rows). Run after the public repo was created (2024-08-30).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v2-5-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.7242,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2025-05-01",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik-11B-v2.5-Instruct', model path speakleash/Bielik-11B-v2.5-Instruct; eq-bench_v2_pl score 72.42 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-05-01 19:27:42. Identity: catalogue bielik-11b-v2-5-instruct (HF speakleash/Bielik-11B-v2.5-Instruct). SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab (as for the Bielik CPTUB rows). Run after the public repo was created (2025-05-01 09:10 UTC).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-4-5b-v3-0-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.5621,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2025-05-04",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik-4.5B-v3.0-Instruct', model path speakleash/Bielik-4.5B-v3.0-Instruct; eq-bench_v2_pl score 56.21 (divided by 100); 163.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-05-04 16:16:42. Identity: catalogue bielik-4-5b-v3-0-instruct (HF speakleash/Bielik-4.5B-v3.0-Instruct). SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab (as for the Bielik CPTUB rows). Run after the public repo was created (2025-04-18).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v3-0-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.712,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2025-12-21",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik-11B-v3.0-Instruct', model path speakleash/Bielik-11B-v3.0-Instruct; eq-bench_v2_pl score 71.2 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-12-21 14:47:06. Identity: catalogue bielik-11b-v3-0-instruct (HF speakleash/Bielik-11B-v3.0-Instruct). SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab (as for the Bielik CPTUB rows). Run after the public repo was created (2025-11-07).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "openchat-3-5-0106-gemma",
    "bench": "polish-eq-bench-v2",
    "score": 0.5862,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'openchat/openchat-3.5-0106-gemma', model path openchat/openchat-3.5-0106-gemma; eq-bench_v2_pl score 58.62 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 17:30:31. Latest of 5 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 60.11 (2024-06-19), 60.11 (2024-06-19), 59.93 (2024-06-19), 59.93 (2024-06-19). Identity: catalogue openchat-3-5-0106-gemma (HF openchat/openchat-3.5-0106-gemma).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "nous-hermes-2-solar-10-7b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4985,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'NousResearch/Nous-Hermes-2-SOLAR-10.7B', model path NousResearch/Nous-Hermes-2-SOLAR-10.7B; eq-bench_v2_pl score 49.85 (divided by 100); 169.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 23:51:07. Latest of 2 unquantised eq-bench_v2_pl runs for this path; earlier run(s) not imported: 48.22 (2024-06-19). Identity: catalogue nous-hermes-2-solar-10-7b (HF NousResearch/Nous-Hermes-2-SOLAR-10.7B).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "hermes-2-pro-llama-3-8b",
    "bench": "polish-eq-bench-v2",
    "score": 0.5457,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'NousResearch/Hermes-2-Pro-Llama-3-8B', model path NousResearch/Hermes-2-Pro-Llama-3-8B; eq-bench_v2_pl score 54.57 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 21:18:18. Identity: catalogue hermes-2-pro-llama-3-8b (HF NousResearch/Hermes-2-Pro-Llama-3-8B).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "hermes-2-theta-llama-3-8b",
    "bench": "polish-eq-bench-v2",
    "score": 0.5488,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'NousResearch/Hermes-2-Theta-Llama-3-8B', model path NousResearch/Hermes-2-Theta-Llama-3-8B; eq-bench_v2_pl score 54.88 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 21:25:22. Identity: catalogue hermes-2-theta-llama-3-8b (HF NousResearch/Hermes-2-Theta-Llama-3-8B).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "falcon2-11b",
    "bench": "polish-eq-bench-v2",
    "score": 0.4241,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'tiiuae/falcon-11B', model path tiiuae/falcon-11B; eq-bench_v2_pl score 42.41 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 01:23:54. Identity: catalogue falcon2-11b (HF tiiuae/falcon-11B).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "trurl-2-13b-academic",
    "bench": "polish-eq-bench-v2",
    "score": 0.2592,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Voicelab/trurl-2-13b-academic', model path Voicelab/trurl-2-13b-academic; eq-bench_v2_pl score 25.92 (divided by 100); 162.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 02:38:04. Identity: catalogue trurl-2-13b-academic (HF Voicelab/trurl-2-13b-academic).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "internlm2-chat-20b",
    "bench": "polish-eq-bench-v2",
    "score": 0.3652,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'internlm/internlm2-chat-20b', model path internlm/internlm2-chat-20b; eq-bench_v2_pl score 36.52 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 09:47:11. Identity: catalogue internlm2-chat-20b (HF internlm/internlm2-chat-20b).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "qwen2-57b-a14b-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.5764,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Qwen/Qwen2-57B-A14B-Instruct', model path Qwen/Qwen2-57B-A14B-Instruct; eq-bench_v2_pl score 57.64 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 16:19:41. Identity: catalogue qwen2-57b-a14b-instruct (HF Qwen/Qwen2-57B-A14B-Instruct).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "llama-3-70b-base",
    "bench": "polish-eq-bench-v2",
    "score": 0.461,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "independent",
    "date": "2024-06-20",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'meta-llama/Meta-Llama-3-70B', model path meta-llama/Meta-Llama-3-70B; eq-bench_v2_pl score 46.1 (divided by 100); 145.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-20 16:59:30. Identity: catalogue llama-3-70b-base (HF meta-llama/Meta-Llama-3-70B). Base (pretrained) checkpoint; 145 of 171 answers parseable, above the harness minimum of 83%.",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "hindigen-v1-3c3h",
    "score": 0.7045,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json",
    "by": "independent",
    "date": "2026-04-06",
    "note": "Inception HindiGen-v1 entry 'meta-llama/Llama-3.1-70B-Instruct'; 3C3H Score 0.7045 (95% CI 0.652239-0.755251, n=259); judge claude-3-7-sonnet-20250219; 259/279 entries answered; precision bfloat16. No run date in the file; date is the pinned Space revision date. Identity: catalogue llama-3-1-70b (HF meta-llama/Llama-3.1-70B-Instruct); the same entry's AraGen-v3 and Arabic IFEval rows are already imported.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "mpt-7b",
    "bench": "arc-challenge",
    "score": 0.4189419795221843,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "arc-easy",
    "score": 0.7003367003367004,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "boolq",
    "score": 0.7394495412844037,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "hellaswag",
    "score": 0.7617008564031069,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "lambada-openai",
    "score": 0.6863962740151368,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "openbookqa",
    "score": 0.314,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "piqa",
    "score": 0.7889009793253536,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "sciq",
    "score": 0.937,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "social-iqa",
    "score": 0.45138178096212894,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "winogrande",
    "score": 0.6803472770323599,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mpt-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.3348523259251629,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/mosaicml-mpt-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.truthfulqa_mc.mc2; task version 1; model config use_fast=True,pretrained=mosaicml/mpt-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy, TruthfulQA uses mc2. StableLM README describes df3da98/add-siqa for Alpha and lm-bench for 4E1T; no run date published (date is repository snapshot). Identity: catalogue mpt-7b (MPT-7B base); model config pretrained=mosaicml/mpt-7b, MosaicML's own repository.",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-2-7b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.5275,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-7b-hf', instruction-tuned (📝), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '37.06 ± 1.97 / 52.75 ± 1.81'. The leaderboard lists this base checkpoint as 📝. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-2-7b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.3125,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-7b-hf', instruction-tuned (📝), default setting; EuroEval version 9.2.0. Cell 'MCC / accuracy' = '11.77 ± 1.81 / 31.25 ± 1.99'. The leaderboard lists this base checkpoint as 📝. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "llama-2-13b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.7395,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'meta-llama/Llama-2-13b-hf', base model (🧠), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '64.96 ± 1.29 / 73.95 ± 1.03'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-small",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2279,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2', base model (🧠), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '-1.53 ± 1.17 / 22.79 ± 2.61'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-small",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2472,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2', base model (🧠), default setting; EuroEval version 15.4.1. Cell 'MCC / accuracy' = '-0.29 ± 1.13 / 24.72 ± 0.35'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-medium",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2586,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2-medium', base model (🧠), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '-0.38 ± 1.96 / 25.86 ± 2.48'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-medium",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2488,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2-medium', base model (🧠), default setting; EuroEval version 15.4.1. Cell 'MCC / accuracy' = '-0.17 ± 0.33 / 24.88 ± 0.47'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-large",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2652,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2-large', base model (🧠), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '-1.86 ± 2.62 / 26.52 ± 3.06'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gpt-2-large",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2503,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'openai-community/gpt2-large', base model (🧠), default setting; EuroEval version 15.4.1. Cell 'MCC / accuracy' = '-0.07 ± 0.57 / 25.03 ± 0.52'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gemma-3-270m",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2672,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-270m-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0. Cell 'MCC / accuracy' = '-1.50 ± 2.29 / 26.72 ± 2.63'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gemma-3-270m",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2517,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-3-270m-it', instruction-tuned (📝), default setting; EuroEval version 16.9.0.dev0. Cell 'MCC / accuracy' = '0.69 ± 1.22 / 25.17 ± 1.03'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gemma-2b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2762,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2b-it', instruction-tuned (📝), default setting; EuroEval version 15.8.2. Cell 'MCC / accuracy' = '4.55 ± 1.03 / 27.62 ± 0.78'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gemma-2b-base",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2685,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-2b', base model (🧠), default setting; EuroEval version 15.8.2. Cell 'MCC / accuracy' = '5.43 ± 1.02 / 26.85 ± 0.46'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "gemma-7b",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2651,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'google/gemma-7b-it', instruction-tuned (📝), default setting; EuroEval version 15.6.1. Cell 'MCC / accuracy' = '4.67 ± 1.97 / 26.51 ± 1.10'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "euroeval-en-hellaswag-test-5shot",
    "score": 0.2498,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset hellaswag: HellaSwag (common-sense, 4-option MC), test split (2048 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'microsoft/Phi-4-reasoning-plus', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '0.57 ± 1.03 / 24.98 ± 0.53'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "deepseek-r1-distill-qwen-1-5b",
    "bench": "euroeval-en-life-in-the-uk-test-5shot",
    "score": 0.2994,
    "source": "https://github.com/EuroEval/leaderboards/blob/1a77f1d2f8ba1ba05ab73287fb090d76bf41b2e0/leaderboards/english_all.csv",
    "by": "independent",
    "date": "2026-04-17",
    "note": "EuroEval English generative leaderboard, leaderboards/english_all.csv @1a77f1d (english_all.json: 'Last updated: 2026-04-17 17:12:00 CET'); dataset life_in_the_uk: Life in the UK (knowledge, 4-option MC), test split (512 questions), 5-shot (EuroEval default; not zero-shot), answered with a letter (EuroEval prompt templates); score = accuracy, EuroEval's secondary metric (primary is MCC), mean over EuroEval's 10 evaluation iterations, divided by 100; entry 'deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', reasoning model (🤔), default setting; EuroEval version 15.10.1.dev0. Cell 'MCC / accuracy' = '6.48 ± 1.81 / 29.94 ± 1.35'. Held at the 2026-10-06 import (near chance / historical bridge); imported 2026-10-11 under the include-every-result rule.",
    "source_id": "euroeval-leaderboards"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "legal-research-bench-allpass",
    "score": 0.429,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Claude Opus 4.8', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 42.9; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "claude-opus-4-8",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.087,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Claude Opus 4.8', 'All-pass (1-shot)' = 8.7 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gpt-5-5",
    "bench": "legal-research-bench-allpass",
    "score": 0.4,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'GPT-5.5', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 40.0; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gpt-5-5",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.068,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'GPT-5.5', 'All-pass (1-shot)' = 6.8 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "legal-research-bench-allpass",
    "score": 0.385,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Claude Sonnet 4.6', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 38.5; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.073,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Claude Sonnet 4.6', 'All-pass (1-shot)' = 7.3 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "glm-5-2",
    "bench": "legal-research-bench-allpass",
    "score": 0.312,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'GLM-5.2', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 31.2; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "glm-5-2",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.034,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'GLM-5.2', 'All-pass (1-shot)' = 3.4 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "legal-research-bench-allpass",
    "score": 0.308,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Gemini 3.5 Flash', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 30.8; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gemini-3-5-flash",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.063,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Gemini 3.5 Flash', 'All-pass (1-shot)' = 6.3 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "minimax-m3",
    "bench": "legal-research-bench-allpass",
    "score": 0.298,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'MiniMax-M3', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 29.8; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "minimax-m3",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.029,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'MiniMax-M3', 'All-pass (1-shot)' = 2.9 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "glm-5-1",
    "bench": "legal-research-bench-allpass",
    "score": 0.276,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'GLM-5.1', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 27.6; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "glm-5-1",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.031,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'GLM-5.1', 'All-pass (1-shot)' = 3.1 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "qwen3-7-max",
    "bench": "legal-research-bench-allpass",
    "score": 0.252,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Qwen3.7-Max', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 25.2; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "qwen3-7-max",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.027,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Qwen3.7-Max', 'All-pass (1-shot)' = 2.7 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "legal-research-bench-allpass",
    "score": 0.232,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'DeepSeek-V4-Pro', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 23.2; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1). 'DeepSeek-V4-Pro' mapped to the April 2026 preview: the paper says every evaluated model was released between February and June 2026, and the GA checkpoint came out in August 2026.",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.041,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'DeepSeek-V4-Pro', 'All-pass (1-shot)' = 4.1 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1). 'DeepSeek-V4-Pro' mapped to the April 2026 preview: the paper says every evaluated model was released between February and June 2026, and the GA checkpoint came out in August 2026.",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "legal-research-bench-allpass",
    "score": 0.208,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Gemini 3.1 Pro Preview', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 20.8; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.065,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Gemini 3.1 Pro Preview', 'All-pass (1-shot)' = 6.5 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "kimi-k2-6",
    "bench": "legal-research-bench-allpass",
    "score": 0.155,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Kimi K2.6', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 15.5; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "kimi-k2-6",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.029,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Kimi K2.6', 'All-pass (1-shot)' = 2.9 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "grok-4-3",
    "bench": "legal-research-bench-allpass",
    "score": 0.153,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'Grok 4.3', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 15.3; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "grok-4-3",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.041,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'Grok 4.3', 'All-pass (1-shot)' = 4.1 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "legal-research-bench-allpass",
    "score": 0.126,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 3, row 'GPT-5.4-mini', All-pass (%) printed as mean ± s.e.; also Table 4 'All-pass (tools)' = 12.6; stored /100. Agentic harness: Tavily web search, CourtListener case-law search, page parsing, retrieval and submit tools; 3-hour wall-clock cap per question. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "legal-research-bench-no-tools-allpass",
    "score": 0.022,
    "source": "https://arxiv.org/abs/2610.00609v1",
    "by": "independent",
    "date": "2026-09-30",
    "note": "Legal Research Bench (arXiv:2610.00609v1, CC BY 4.0) Table 4 (tool ablation), row 'GPT-5.4-mini', 'All-pass (1-shot)' = 2.2 (%); single-shot answer with no tool access; stored /100. Authors' own runs (Vals AI): 'We evaluate thirteen frontier models using this harness'; every model answers all 413 questions, one run per question, graded by a GPT-5.4 judge validated against attorneys. Reasoning effort / thinking setting not stated in the paper. Models 'all released between February and June 2026' (Sec. 4.1).",
    "source_id": "paper-2610.00609v1-legal-research-bench"
  },
  {
    "model": "o3",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.468,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'o3', 'Acc. (Class-Balanced)' = 46.8 ± 2.2 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/o3-2025-04-16 (the only o3 snapshot). The harness sends no reasoning_effort (llm.py @ac07b92), so the API's default effort applies; Table 2 Reasoning = ✓.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "claude-3-7-sonnet-20250219",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.459,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Claude 3.7 Sonnet (Thinking)', 'Acc. (Class-Balanced)' = 45.9 ± 2.2 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): anthropic/claude-3-7-sonnet-20250219-thinking: extended thinking enabled with budget_tokens 13384, max_tokens 16384 (llm.py @ac07b92). Highest reasoning row printed; the non-thinking 'Claude 3.7 Sonnet' row (44.3) is not imported.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "o4-mini",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.373,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'o4 Mini', 'Acc. (Class-Balanced)' = 37.3 ± 2.2 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/o4-mini-2025-04-16 (the only o4-mini snapshot); no reasoning_effort sent, so the API's default effort applies.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-4-1",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.267,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'GPT 4.1', 'Acc. (Class-Balanced)' = 26.7 ± 2.0 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/gpt-4.1-2025-04-14 (the only GPT-4.1 snapshot); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "grok-3-beta",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.258,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Grok 3 Beta', 'Acc. (Class-Balanced)' = 25.8 ± 1.9 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): grok/grok-3-beta (xAI API model grok-3-beta); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.214,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'o1', 'Acc. (Class-Balanced)' = 21.4 ± 1.7 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/o1-2024-12-17 (dated snapshot; not o1-preview); no reasoning_effort sent, so the API's default effort applies.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.203,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'GPT 4.1 mini', 'Acc. (Class-Balanced)' = 20.3 ± 1.7 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/gpt-4.1-mini-2025-04-14 (the only snapshot); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-4o-2024-08-06",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.2,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'GPT 4o (2024-08-06)', 'Acc. (Class-Balanced)' = 20.0 ± 1.7 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/gpt-4o-2024-08-06 (dated snapshot as printed). Table 2 marks it Reasoning ✓, apparently a typo; GPT-4o has no reasoning setting.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.144,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Gemini 2.0 Flash (001)', 'Acc. (Class-Balanced)' = 14.4 ± 1.4 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): google/gemini-2.0-flash-001 (stable version as printed); no thinking.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.131,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Claude 3.5 Haiku Latest', 'Acc. (Class-Balanced)' = 13.1 ± 1.4 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): README lists anthropic/claude-3-5-haiku-20241022, the only Claude 3.5 Haiku snapshot (the 'latest' alias pointed to it); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "o3-mini",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.128,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'o3 Mini', 'Acc. (Class-Balanced)' = 12.8 ± 1.4 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/o3-mini-2025-01-31 (the only o3-mini snapshot); no reasoning_effort sent, so the API's default effort applies.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.108,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'GPT 4o Mini', 'Acc. (Class-Balanced)' = 10.8 ± 1.2 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/gpt-4o-mini-2024-07-18 (the only gpt-4o-mini snapshot); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "mistral-small-3-1",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.108,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Mistral Small 3.1 (03/2025)', 'Acc. (Class-Balanced)' = 10.8 ± 1.3 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): mistralai/mistral-small-2503 = Mistral Small 3.1 (25.03); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "llama-4-scout",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.058,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'LLaMA 4 Scout', 'Acc. (Class-Balanced)' = 5.8 ± 1.0 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): together/meta-llama/Llama-4-Scout-17B-16E-Instruct (served by Together AI); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "command-a",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.046,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Command A', 'Acc. (Class-Balanced)' = 4.6 ± 0.9 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): cohere/command-a-03-2025; non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "llama-4-maverick",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.031,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'LLaMA 4 Maverick', 'Acc. (Class-Balanced)' = 3.1 ± 0.8 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 (Meta's official FP8 release of the instruct model, served by Together AI; same mapping as other FP8 Maverick rows in the catalogue); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.028,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'LLaMA 3.3 Instruct Turbo (70B)', 'Acc. (Class-Balanced)' = 2.8 ± 0.8 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): together/meta-llama/Llama-3.3-70B-Instruct-Turbo (Together AI's serving of Llama 3.3 70B Instruct; same mapping as the ContractBench and Galileo rows); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.024,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'GPT 4.1 nano', 'Acc. (Class-Balanced)' = 2.4 ± 0.7 (%); stored /100. Class-balanced accuracy is the paper's headline metric (Sec. 4.2: 'We find the class-balanced accuracy to be more representative of the model's general agentic financial capabilities'); naive accuracy on the same answers not imported. All 537 questions ('All metrics reported in this paper were calculated on the complete 537 samples'). Agent harness with Google search (SerpAPI), EDGAR search, HTML parsing and retrieval tools; identical prompt, temperature 0 where supported, 16,384 max tokens; rubric LLM-judge grading. Authors' own runs (Vals AI): 'All models were evaluated under consistent conditions to ensure a fair comparison'. Model id from the harness README 'Available Models' list at vals-ai/finance-agent@ac07b92 (2025-05-16, four days before submission; 'This repo contains the codebase to run the agent that was used to create the benchmark'): openai/gpt-4.1-nano-2025-04-14 (the only snapshot); non-reasoning model.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "gpt-5-3-codex",
    "bench": "vibe-code-bench-test",
    "score": 0.6177,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'GPT-5.3-Codex', Accuracy = 61.77 ± 4.71%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id openai/gpt-5.3-codex-api-preview (API preview access before the paper's 2026-03-04 v1), reasoning xhigh.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "gpt-5-2",
    "bench": "vibe-code-bench-test",
    "score": 0.535,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'GPT-5.2', Accuracy = 53.50 ± 5.07%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id openai/gpt-5.2, reasoning xhigh.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "claude-opus-4-6",
    "bench": "vibe-code-bench-test",
    "score": 0.535,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Claude Opus 4.6 Thinking', Accuracy = 53.50 ± 4.68%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id anthropic/claude-opus-4-6-thinking, temperature 1.0. Highest reasoning row printed for this model; the plain 'Claude Opus 4.6' row (anthropic/claude-opus-4-6, 57.57 ± 4.37) is not imported.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "vibe-code-bench-test",
    "score": 0.5148,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Claude Sonnet 4.6', Accuracy = 51.48 ± 4.64%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id anthropic/claude-sonnet-4-6, temperature 1.0; no thinking/effort setting printed (only row for this model).",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "gpt-5-2-codex",
    "bench": "vibe-code-bench-test",
    "score": 0.3791,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'GPT-5.2-Codex', Accuracy = 37.91 ± 4.58%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id openai/gpt-5.2-codex, reasoning high (the only setting printed).",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "gemini-3-1-pro",
    "bench": "vibe-code-bench-test",
    "score": 0.3203,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Gemini 3.1 Pro', Accuracy = 32.03 ± 4.34%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id google/gemini-3.1-pro-preview, reasoning high.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "glm-5",
    "bench": "vibe-code-bench-test",
    "score": 0.2336,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'GLM-5', Accuracy = 23.36 ± 4.03%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id zai/glm-5, temperature 1.0, top-p 0.95, vision disabled ('Some provider configurations did not expose usable browser-vision support'); no reasoning setting printed.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "vibe-code-bench-test",
    "score": 0.202,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Gemini 3 Flash', Accuracy = 20.20 ± 3.95%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id google/gemini-3-flash-preview, reasoning high.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "kimi-k2-5",
    "bench": "vibe-code-bench-test",
    "score": 0.1754,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Kimi-K2.5 Thinking', Accuracy = 17.54 ± 3.26%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id kimi/kimi-k2.5-thinking (thinking mode), temperature 1.0.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "qwen3-5-plus",
    "bench": "vibe-code-bench-test",
    "score": 0.1574,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Qwen3.5 Plus Thinking', Accuracy = 15.74 ± 3.18%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id alibaba/qwen3.5-plus-thinking (hosted Qwen3.5-Plus, thinking mode), temperature 0.6. Table 4 marks it open-weight; the catalogue keeps the hosted qwen3.5-plus separate from the open 397B-A17B weights, and this run used the hosted API model.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "minimax-m2-5",
    "bench": "vibe-code-bench-test",
    "score": 0.1485,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'MiniMax M2.5', Accuracy = 14.85 ± 2.95%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id minimax/MiniMax-M2.5, temperature 1.0, vision disabled; no reasoning setting printed.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "gpt-5-mini",
    "bench": "vibe-code-bench-test",
    "score": 0.1417,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'GPT-5 Mini', Accuracy = 14.17 ± 3.54%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id openai/gpt-5-mini-2025-08-07, reasoning high.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "vibe-code-bench-test",
    "score": 0.1139,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Claude Haiku 4.5 Thinking', Accuracy = 11.39 ± 3.13%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id anthropic/claude-haiku-4-5-20251001-thinking (extended thinking), temperature 1.0.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "vibe-code-bench-test",
    "score": 0.0511,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'DeepSeek V3.2 Thinking', Accuracy = 5.11 ± 2.13%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id deepseek-v3p2-thinking via Fireworks (DeepSeek-V3.2 open weights, thinking mode), reasoning high, vision disabled.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "vibe-code-bench-test",
    "score": 0.012,
    "source": "https://arxiv.org/abs/2603.04601v3",
    "by": "independent",
    "date": "2026-05-13",
    "note": "Vibe Code Bench (arXiv:2603.04601v3, CC BY 4.0) Table 4, row 'Grok 4.1 Fast Reasoning', Accuracy = 1.20 ± 1.20%; stored /100. Accuracy = share of browser workflows passed (pass = at least 90% of substeps succeed, Browser Use evaluator with Claude Sonnet 4.5), mean over the 50 test-split apps, printed as mean ± s.e.; Table 4 identical in v1-v3. OpenHands-based harness with terminal, browser, Supabase/MailHog/Stripe and Tavily, 5-hour wall-clock budget per app. Authors' own runs (Vals AI): 'We evaluate 16 models from 9 providers'; 'All models use vision if supported, the highest reasoning level available for that model, and the default temperature specified by the model provider.' Table 15 id grok/grok-4-1-fast-reasoning, temperature 0.7.",
    "source_id": "paper-2603.04601v3-vibe-code-bench"
  },
  {
    "model": "claude-opus-4-5",
    "bench": "formalproofbench",
    "score": 0.335,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'Claude Opus 4.5 (Thinking)', Acc. (%) = 33.50; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Thinking enabled as labelled (budget not stated); native Anthropic API.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "formalproofbench",
    "score": 0.185,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'Gemini 3 Pro (11/25)', Acc. (%) = 18.50; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). '11/25' = the November 2025 Gemini 3 Pro preview (API gemini-3-pro-preview); native Google API, provider-default settings.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "formalproofbench",
    "score": 0.18,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'Claude Sonnet 4.5 (Thinking)', Acc. (%) = 18.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Thinking enabled as labelled (budget not stated); native Anthropic API.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gpt-5",
    "bench": "formalproofbench",
    "score": 0.17,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'GPT 5', Acc. (%) = 17.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). GPT-5 (gpt-5-2025-08-07, the only snapshot); native OpenAI API, provider-default reasoning effort.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "formalproofbench",
    "score": 0.15,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'Gemini 3 Flash (12/25)', Acc. (%) = 15.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). '12/25' = the December 2025 Gemini 3 Flash preview (API gemini-3-flash-preview); native Google API, provider-default settings.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gpt-5-2",
    "bench": "formalproofbench",
    "score": 0.135,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'GPT 5.2', Acc. (%) = 13.50; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Native OpenAI API, provider-default reasoning effort (not printed).",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gpt-5-nano",
    "bench": "formalproofbench",
    "score": 0.125,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'GPT 5 Nano', Acc. (%) = 12.50; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Native OpenAI API, provider-default reasoning effort.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "deepseek-v3-2",
    "bench": "formalproofbench",
    "score": 0.03,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'DeepSeek V3.2 (Thinking)', Acc. (%) = 3.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Thinking mode is the highest reasoning row printed for this model; the 'DeepSeek V3.2 (Nonthinking)' row (8.00) is not imported. Native DeepSeek API, which served V3.2 from December 2025 (the table includes the December 2025 Gemini 3 Flash, so runs postdate it).",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "gpt-5-mini",
    "bench": "formalproofbench",
    "score": 0.07,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'GPT 5 Mini', Acc. (%) = 7.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Native OpenAI API, provider-default reasoning effort.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "glm-4-7",
    "bench": "formalproofbench",
    "score": 0.045,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'GLM 4.7', Acc. (%) = 4.50; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Native Zhipu AI API; default settings.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "grok-4-1-fast-reasoning",
    "bench": "formalproofbench",
    "score": 0.04,
    "source": "https://arxiv.org/abs/2603.26996v1",
    "by": "independent",
    "date": "2026-03-27",
    "note": "FormalProofBench (arXiv:2603.26996v1, CC BY 4.0) Table 2, row 'Grok 4.1 Fast (Reasoning)', Acc. (%) = 4.00; stored /100. 200 problems, Lean 4 kernel-checked proofs, agent harness with lean_loogle, lean_run_code and submit_proof, up to 40 turns. Authors' own runs (Vals AI and co-authors): Sec. 3.2 'We standardize our evaluation settings by using the default sampling settings from each model provider, and by using the native model provider rather than a third-party inference provider.' Reasoning effort beyond the printed label is not stated (provider defaults). Reasoning variant as labelled (xAI API grok-4-1-fast-reasoning); native xAI API.",
    "source_id": "paper-2603.26996v1-formalproofbench"
  },
  {
    "model": "grok-3-mini-beta",
    "bench": "finance-agent-benchmark-v1-class-balanced",
    "score": 0.309,
    "source": "https://arxiv.org/abs/2508.00828v1",
    "by": "independent",
    "date": "2025-05-20",
    "note": "Finance Agent Benchmark (arXiv:2508.00828v1, CC BY 4.0) Table 2, row 'Grok 3 Mini Fast High Reasoning', 'Acc. (Class-Balanced)' = 30.9 ± 1.9 (%); stored /100. Harness llm.py @ac07b92 calls model='grok-3-mini-fast-beta' with reasoning_effort='high'. Mapped to grok-3-mini-beta: the fast tier serves the same model on faster infrastructure (Oracle Cloud's xAI page: 'The xai.grok-3-mini and xai.grok-3-mini-fast models, both use the same underlying model and deliver identical response quality'; that statement covers the GA ids). Owner rule 2026-10-11: only licensing holds rows back.",
    "source_id": "paper-2508.00828v1-finance-agent-benchmark"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "deepswe-v1-1",
    "score": 0.587,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'DeepSWE 1.1', column 'Qwen3.8-Flash' = 58.7; footnote 1: 'evaluated with the Claude Code and mini-SWE-agent harnesses, temp=1.0, top_p=0.95, 256K context window. We report the highest score across the two harnesses'. Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.625,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Pro', column 'Qwen3.8-Flash' = 62.5; footnote 2: 'all models are evaluated with the Claude Code harness ... Problematic tasks were corrected and all baseline models were re-evaluated on the refined benchmark'. Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "swe-bench-multilingual",
    "score": 0.81,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'SWE-bench Multilingual', column 'Qwen3.8-Flash' = 81.0; footnote 3: 'evaluated with the mini-SWE-agent harness, temp=1.0, top_p=0.95, 256K context window'. Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "ifbench-unspecified",
    "score": 0.813,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'IFBench', column 'Qwen3.8-Flash' = 81.3; Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "gpqa-diamond",
    "score": 0.917,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'GPQA Diamond', column 'Qwen3.8-Flash' = 91.7; Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "hle",
    "score": 0.359,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'HLE', column 'Qwen3.8-Flash' = 35.9; footnote 6: 'judged by GPT-4o'; no tools stated. Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.919,
    "source": "https://qwen.ai/blog?id=qwen3.8-omni-flash",
    "by": "lab",
    "date": "2026-09-18",
    "note": "Qwen3.8-Omni-Flash launch post 'Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.' (Qwen Team, 2026-09-18; read via the page's own article API qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-omni-flash&type=qwen_ai, re-read 2026-10-11), text-benchmark table (columns Qwen3.8-Omni-Flash | Qwen3.8-Flash | Qwen3.8-27B | Qwen3.7-Plus | DeepSeek-V4-Flash-0731 | Claude-Opus-4.6 (Max)), row 'LiveCodeBench v6', column 'Qwen3.8-Flash' = 91.9; date window not printed. Identity: the column header 'Qwen3.8-Flash' is the open-weight Qwen3.8-Flash-Next (catalogue qwen3-8-flash-next): footnote 1 under this table says 'notably, Qwen3.8-Flash-Next performs best on mini-SWE-agent', and the same column's CharXiv (RQ) cell in the vision table (84.6 without CI / 90.6 with CI) equals the pinned Qwen3.8-Flash-Next card; no other Qwen3.8-Flash checkpoint is catalogued. Same-lab column in Qwen's own table (rule b); reasoning setting not printed. Held in pass 8 for the header/identity question; imported under the 2026-10-11 owner decision.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "swe-bench-verified",
    "score": 0.6,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "independent",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86, re-read 2026-10-11), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'SWE Verified', column 'Qwen3.5-9B' = 60.0; metric avg@3. Qwen3.5-9B is the base model the SFT checkpoint was trained from ('supervised fine-tuning of Qwen3.5-9B'). Treated as Xiaomi's own measurement: the same column carries cells on Xiaomi's internal sets (MiMo Code (mini) 19.5, MiMo Cyber (mini) 5.7, MiMo General (mini) 28.5, MiMo Visual Coding (mini) 61.7; '† Internal evaluation sets') that only Xiaomi can have measured, and uses Xiaomi's avg@3/avg@1 protocol; no explicit 'we evaluated the baselines' sentence. Held in pass 8 for that missing sentence; imported under the 2026-10-11 owner decision. Thinking mode for the baseline not stated.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.32,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "independent",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86, re-read 2026-10-11), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'SWE Pro', column 'Qwen3.5-9B' = 32.0; metric avg@3; split (public/commercial) not stated. Qwen3.5-9B is the base model the SFT checkpoint was trained from ('supervised fine-tuning of Qwen3.5-9B'). Treated as Xiaomi's own measurement: the same column carries cells on Xiaomi's internal sets (MiMo Code (mini) 19.5, MiMo Cyber (mini) 5.7, MiMo General (mini) 28.5, MiMo Visual Coding (mini) 61.7; '† Internal evaluation sets') that only Xiaomi can have measured, and uses Xiaomi's avg@3/avg@1 protocol; no explicit 'we evaluated the baselines' sentence. Held in pass 8 for that missing sentence; imported under the 2026-10-11 owner decision. Thinking mode for the baseline not stated.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "automationbench-v1-0-6",
    "score": 0.05,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "independent",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86, re-read 2026-10-11), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'AutomationBench v1.0.6', column 'Qwen3.5-9B' = 5.0; metric avg@1. Qwen3.5-9B is the base model the SFT checkpoint was trained from ('supervised fine-tuning of Qwen3.5-9B'). Treated as Xiaomi's own measurement: the same column carries cells on Xiaomi's internal sets (MiMo Code (mini) 19.5, MiMo Cyber (mini) 5.7, MiMo General (mini) 28.5, MiMo Visual Coding (mini) 61.7; '† Internal evaluation sets') that only Xiaomi can have measured, and uses Xiaomi's avg@3/avg@1 protocol; no explicit 'we evaluated the baselines' sentence. Held in pass 8 for that missing sentence; imported under the 2026-10-11 owner decision. Thinking mode for the baseline not stated.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "terminal-bench-2.1",
    "score": 0.27,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "independent",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86, re-read 2026-10-11), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'Terminal Bench 2.1', column 'Qwen3.5-9B' = 27.0; metric avg@1. Qwen3.5-9B is the base model the SFT checkpoint was trained from ('supervised fine-tuning of Qwen3.5-9B'). Treated as Xiaomi's own measurement: the same column carries cells on Xiaomi's internal sets (MiMo Code (mini) 19.5, MiMo Cyber (mini) 5.7, MiMo General (mini) 28.5, MiMo Visual Coding (mini) 61.7; '† Internal evaluation sets') that only Xiaomi can have measured, and uses Xiaomi's avg@3/avg@1 protocol; no explicit 'we evaluated the baselines' sentence. Held in pass 8 for that missing sentence; imported under the 2026-10-11 owner decision. Thinking mode for the baseline not stated.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "toolathlon-verified",
    "score": 0.259,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B/blob/2367e865d009c13ac81713a2878291d33ab28177/README.md",
    "by": "independent",
    "date": "2026-09-21",
    "note": "Xiaomi MiMo-V2.6-Distill-Qwen-9B model card (MIT, pinned 2367e86, re-read 2026-10-11), '## Evaluation' table ('Results for the released SFT checkpoint, as reported in the MiMo-V2.6 technical report'), row 'Toolathlon-Verified', column 'Qwen3.5-9B' = 25.9; metric avg@1. Qwen3.5-9B is the base model the SFT checkpoint was trained from ('supervised fine-tuning of Qwen3.5-9B'). Treated as Xiaomi's own measurement: the same column carries cells on Xiaomi's internal sets (MiMo Code (mini) 19.5, MiMo Cyber (mini) 5.7, MiMo General (mini) 28.5, MiMo Visual Coding (mini) 61.7; '† Internal evaluation sets') that only Xiaomi can have measured, and uses Xiaomi's avg@3/avg@1 protocol; no explicit 'we evaluated the baselines' sentence. Held in pass 8 for that missing sentence; imported under the 2026-10-11 owner decision. Thinking mode for the baseline not stated.",
    "source_id": "card3-xiaomimimo-mimo-v2-6-distill-qwen-9b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "aime-2025",
    "score": 0.5187,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'AIME25', column 'LFM2.5-2.6B (2.6B)' = 51.87; divided by 100; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "aime-2025",
    "score": 0.2633,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'AIME25', column 'gemma-4-E2B-it (5.1B)' = 26.33; divided by 100; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "aime-2025",
    "score": 0.3427,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'AIME25', column 'gemma-4-E4B-it (8B)' = 34.27; divided by 100; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "aime-2025",
    "score": 0.4933,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'AIME25', column 'Qwen3.5-4B (4.7B)' = 49.33; divided by 100; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "aime-2025",
    "score": 0.5607,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'AIME25', column 'Qwen3.5-9B (9.7B)' = 56.07; divided by 100; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.5941,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'LiveCodeBenchv6', column 'LFM2.5-2.6B (2.6B)' = 59.41; divided by 100; date window not printed; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.5492,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'LiveCodeBenchv6', column 'gemma-4-E2B-it (5.1B)' = 54.92; divided by 100; date window not printed; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.6377,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'LiveCodeBenchv6', column 'gemma-4-E4B-it (8B)' = 63.77; divided by 100; date window not printed; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.6085,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'LiveCodeBenchv6', column 'Qwen3.5-4B (4.7B)' = 60.85; divided by 100; date window not printed; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.6986,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'LiveCodeBenchv6', column 'Qwen3.5-9B (9.7B)' = 69.86; divided by 100; date window not printed; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "ifbench-unspecified",
    "score": 0.5917,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'IFBench', column 'LFM2.5-2.6B (2.6B)' = 59.17; divided by 100; aggregation not stated; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "ifbench-unspecified",
    "score": 0.3408,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'IFBench', column 'gemma-4-E2B-it (5.1B)' = 34.08; divided by 100; aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "ifbench-unspecified",
    "score": 0.3924,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'IFBench', column 'gemma-4-E4B-it (8B)' = 39.24; divided by 100; aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "ifbench-unspecified",
    "score": 0.484,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'IFBench', column 'Qwen3.5-4B (4.7B)' = 48.40; divided by 100; aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "ifbench-unspecified",
    "score": 0.5647,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'IFBench', column 'Qwen3.5-9B (9.7B)' = 56.47; divided by 100; aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "multi-if-unspecified",
    "score": 0.8007,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Multi-IF', column 'LFM2.5-2.6B (2.6B)' = 80.07; divided by 100; turn/strictness aggregation not stated; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "multi-if-unspecified",
    "score": 0.6944,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Multi-IF', column 'gemma-4-E2B-it (5.1B)' = 69.44; divided by 100; turn/strictness aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "multi-if-unspecified",
    "score": 0.7735,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Multi-IF', column 'gemma-4-E4B-it (8B)' = 77.35; divided by 100; turn/strictness aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "multi-if-unspecified",
    "score": 0.5567,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Multi-IF', column 'Qwen3.5-4B (4.7B)' = 55.67; divided by 100; turn/strictness aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "multi-if-unspecified",
    "score": 0.6255,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Multi-IF', column 'Qwen3.5-9B (9.7B)' = 62.55; divided by 100; turn/strictness aggregation not stated; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "bfcl-v4",
    "score": 0.5688,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'BFCLv4', column 'LFM2.5-2.6B (2.6B)' = 56.88; divided by 100; Liquid's own BFCL v4 run (not the Berkeley leaderboard file), placed in bfcl-v4 like the other card-reported BFCLv4 cells; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "bfcl-v4",
    "score": 0.3698,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'BFCLv4', column 'gemma-4-E2B-it (5.1B)' = 36.98; divided by 100; Liquid's own BFCL v4 run (not the Berkeley leaderboard file), placed in bfcl-v4 like the other card-reported BFCLv4 cells; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "bfcl-v4",
    "score": 0.4639,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'BFCLv4', column 'gemma-4-E4B-it (8B)' = 46.39; divided by 100; Liquid's own BFCL v4 run (not the Berkeley leaderboard file), placed in bfcl-v4 like the other card-reported BFCLv4 cells; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "bfcl-v4",
    "score": 0.5056,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'BFCLv4', column 'Qwen3.5-4B (4.7B)' = 50.56; divided by 100; Liquid's own BFCL v4 run (not the Berkeley leaderboard file), placed in bfcl-v4 like the other card-reported BFCLv4 cells; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "bfcl-v4",
    "score": 0.6013,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'BFCLv4', column 'Qwen3.5-9B (9.7B)' = 60.13; divided by 100; Liquid's own BFCL v4 run (not the Berkeley leaderboard file), placed in bfcl-v4 like the other card-reported BFCLv4 cells; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "claw-eval-avg",
    "score": 0.6285,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Claw-Eval average (EN)', column 'LFM2.5-2.6B (2.6B)' = 62.85; divided by 100; average score, English set; Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "claw-eval-avg",
    "score": 0.5314,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Claw-Eval average (EN)', column 'gemma-4-E2B-it (5.1B)' = 53.14; divided by 100; average score, English set; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "claw-eval-avg",
    "score": 0.5802,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Claw-Eval average (EN)', column 'gemma-4-E4B-it (8B)' = 58.02; divided by 100; average score, English set; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "claw-eval-avg",
    "score": 0.6228,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Claw-Eval average (EN)', column 'Qwen3.5-4B (4.7B)' = 62.28; divided by 100; average score, English set; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "claw-eval-avg",
    "score": 0.6653,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'Claw-Eval average (EN)', column 'Qwen3.5-9B (9.7B)' = 66.53; divided by 100; average score, English set; comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "lfm2-5-2-6b",
    "bench": "tau3-banking",
    "score": 0.0567,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "lab",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'τ³-Bench Banking', column 'LFM2.5-2.6B (2.6B)' = 5.67; divided by 100; tau2-bench grading version and user simulator not stated, so tau3-banking (not -v101); Liquid's own model; the card says 'LFM2.5-2.6B is a pure reasoning model that always thinks before it answers'.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e2b",
    "bench": "tau3-banking",
    "score": 0.0335,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'τ³-Bench Banking', column 'gemma-4-E2B-it (5.1B)' = 3.35; divided by 100; tau2-bench grading version and user simulator not stated, so tau3-banking (not -v101); comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "gemma-4-e4b",
    "bench": "tau3-banking",
    "score": 0.0412,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'τ³-Bench Banking', column 'gemma-4-E4B-it (8B)' = 4.12; divided by 100; tau2-bench grading version and user simulator not stated, so tau3-banking (not -v101); comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-4b",
    "bench": "tau3-banking",
    "score": 0.0545,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'τ³-Bench Banking', column 'Qwen3.5-4B (4.7B)' = 5.45; divided by 100; tau2-bench grading version and user simulator not stated, so tau3-banking (not -v101); comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "qwen3-5-9b",
    "bench": "tau3-banking",
    "score": 0.0515,
    "source": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md#benchmarks",
    "by": "independent",
    "date": "2026-08-24",
    "note": "Liquid AI LFM2.5-2.6B model card (LFM Open License v1.0, pinned 654f946, re-read 2026-10-11), '### Benchmarks' table ('We compared LFM2.5-2.6B with relevant sub-10B models on a diverse suite of benchmarks.'), row 'τ³-Bench Banking', column 'Qwen3.5-9B (9.7B)' = 5.15; divided by 100; tau2-bench grading version and user simulator not stated, so tau3-banking (not -v101); comparator cell treated as Liquid's own run: the card gives no explicit evaluation statement, but every comparator value is printed to two decimals and differs from the makers' own published figures (e.g. Qwen3.5-9B LiveCodeBench v6 69.86 here vs 65.6 on Qwen's card; IFBench 56.47 vs 64.5; gemma-4-E4B-it IFBench 39.24 vs 44 in the Gemma 4 report), so the cells are not copied. Comparator thinking mode not stated. Held in pass 8 for the missing statement; imported under the 2026-10-11 owner decision. The three AA-Omni-Public rows (Artificial Analysis benchmark) and IFStruct, ToolSandbox, PinchBench, BrowseComp+ (no benchmark id) are not used.",
    "source_id": "card3-liquidai-lfm2-5-2-6b"
  },
  {
    "model": "grok-4-0709",
    "bench": "gpqa-diamond",
    "score": 0.87,
    "source": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv",
    "by": "independent",
    "date": "2025-07-09",
    "note": "Epoch AI internal evaluation file gpqa_diamond.csv; snapshot 2026-10-11 (zip sha256 d46e0c8c9975505b58f6fe1ee30716d4acfaf2452ab2a987419cab1f6f06a952); exact model version 'grok-4-0709'; run id A85Zfq2qguE4X9xXBweBHP; published 'Best score (across scorers)' 0.87 (mean_score 0.87, stderr 0.02). This run has no 'Started at', Log viewer or Logs entry (every other internal run in the file has them) and two-decimal values; the zip does not say who ran it. Recorded because it sits in Epoch's CC BY internal file; date set to the model's release date 2025-07-09 since no start date is printed. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "epoch-internal"
  },
  {
    "model": "grok-4-0709",
    "bench": "otis-mock-aime",
    "score": 0.84,
    "source": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv",
    "by": "independent",
    "date": "2025-07-09",
    "note": "Epoch AI internal evaluation file otis_mock_aime_2024_2025.csv; snapshot 2026-10-11 (zip sha256 d46e0c8c9975505b58f6fe1ee30716d4acfaf2452ab2a987419cab1f6f06a952); exact model version 'grok-4-0709'; run id cvTPRDCM38zSTn9Y3MUb9d; published 'Best score (across scorers)' 0.84 (mean_score 0.84, stderr 0.05). This run has no 'Started at', Log viewer or Logs entry and two-decimal values; the zip does not say who ran it. Recorded because it sits in Epoch's CC BY internal file; date set to the model's release date 2025-07-09 since no start date is printed. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "simpleqa-verified",
    "score": 0.46993987975951906,
    "source": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file simpleqa_verified.csv; snapshot 2026-10-11 (zip sha256 d46e0c8c9975505b58f6fe1ee30716d4acfaf2452ab2a987419cab1f6f06a952); exact model version 'deepseek-v4-pro_max' (Epoch record 'DeepSeek-V4-Pro', release date 2026-04-24, separate from its 'deepseek-v4-pro-0813' record); run id Sq9xqkVTmFvELpQy9KH5TM, started 2026-08-27T19:30:29Z; effort max (only published effort for this record in this file); published 'Best score (across scorers)' column; stderr 0.015806515332355317. Identity: mapped to the April preview as Epoch labels it. The run started after the deepseek-v4-pro API id moved to the 0813 weights, but Epoch's 0813 run of the same test 9 seconds earlier (2xfFf6KZAp2gUWYhHGkFgs, already catalogued under deepseek-v4-pro) scored 0.529 with the same effort, about 3.7 standard errors apart, so the two runs answered with different weights; Epoch's own labelling is the better mapping. The inspect log is behind a verification page and could not be read. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "epoch-internal"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "epoch-mystery-games",
    "score": 0.17,
    "source": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv",
    "by": "independent",
    "date": "2026-08-27",
    "note": "Epoch AI internal evaluation; file mystery_game_puzzles.csv; snapshot 2026-10-11 (zip sha256 d46e0c8c9975505b58f6fe1ee30716d4acfaf2452ab2a987419cab1f6f06a952); exact model version 'deepseek-v4-pro_none' (Epoch record 'DeepSeek-V4-Pro', release date 2026-04-24, separate from its 'deepseek-v4-pro-0813' record); run id BmvJaHMCnZzb24ErpAsozU, started 2026-08-27T18:01:08Z; effort none, the only published effort for this record in this file (reasoning off); published 'Best score (across scorers)' column; stderr 0.03775251680686369. Identity: mapped to the April preview as Epoch labels it (same reasoning as its simpleqa_verified run of the same day, which scored far from the 0813 run 9 seconds apart). Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "epoch-internal"
  },
  {
    "model": "qwq-32b",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.1125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/QwQ-32B-Fireworks",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'QwQ-32B-Fireworks' (API model in attempt metadata: 'accounts/fireworks/models/qwq-32b'). Fireworks API, max_tokens 32768, run 2025-03-06. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 45.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.62875,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-haiku-4-5-20251001-thinking-32k",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-haiku-4-5-20251001-thinking-32k' (API model in attempt metadata: 'claude-haiku-4-5-20251001'). extended thinking budget 32000 (highest of 1k/8k/16k/32k), max_tokens 64000, run 2025-10-15/16; 1 null attempt. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 251.5 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "claude-haiku-4-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.051389,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-haiku-4-5-20251001-thinking-32k",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-haiku-4-5-20251001-thinking-32k' (API model in attempt metadata: 'claude-haiku-4-5-20251001'). extended thinking budget 32000 (highest of 1k/8k/16k/32k), run 2025-10-15. ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 6.167 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.7375,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-sonnet-4-5-20250929-thinking-32k",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'claude-sonnet-4-5-20250929-thinking-32k' (API model in attempt metadata: 'claude-sonnet-4-5-20250929'). extended thinking budget 32000 (highest of 1k/8k/16k/32k), max_tokens 64000, run 2025-09-30. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 295.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "claude-sonnet-4-5",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.147222,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-sonnet-4-5-20250929-thinking-32k",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'claude-sonnet-4-5-20250929-thinking-32k' (API model in attempt metadata: 'claude-sonnet-4-5-20250929'). extended thinking budget 32000 (highest), run 2025-09-30. ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 17.667 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.1175,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-4-1-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-4-1-2025-04-14' (API model in attempt metadata: 'gpt-4.1-2025-04-14'). max_completion_tokens 10000, run 2025-04-14. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 47.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-4-1-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-4-1-2025-04-14' (API model in attempt metadata: 'gpt-4.1-2025-04-14'). run 2025-04-14, graded on the 2025-04-04 ARC-AGI-2 eval version (172 test pairs; also 0 on the current version). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 0.0 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.0725,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-4-1-mini-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-4-1-mini-2025-04-14' (API model in attempt metadata: 'gpt-4.1-mini-2025-04-14'). max_completion_tokens 10000, run 2025-04-14. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 29.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-4-1-mini-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-4-1-mini-2025-04-14' (API model in attempt metadata: 'gpt-4.1-mini-2025-04-14'). run 2025-04-14, graded on the 2025-04-04 ARC-AGI-2 eval version (172 test pairs; also 0 on the current version). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 0.0 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.0175,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-4-1-nano-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-4-1-nano-2025-04-14' (API model in attempt metadata: 'gpt-4.1-nano-2025-04-14'). max_completion_tokens 10000, run 2025-04-14. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 7.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-4-1-nano-2025-04-14",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-4-1-nano-2025-04-14' (API model in attempt metadata: 'gpt-4.1-nano-2025-04-14'). run 2025-04-14, graded on the 2025-04-04 ARC-AGI-2 eval version (172 test pairs; also 0 on the current version). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 0.0 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-pro",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.77,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-pro-2025-10-06",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-pro-2025-10-06' (API model in attempt metadata: 'gpt-5-pro-2025-10-06'). default reasoning (Pro has a single high effort), run 2025-10-07..09; 40 of 838 attempts null (failed/timed-out calls). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 308.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-pro",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.133333,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-pro-2025-10-06",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-pro-2025-10-06' (API model in attempt metadata: 'gpt-5-pro-2025-10-06'). run 2025-10-07..09; 24 null attempts and 5 tasks holding only 1 of 2 test pairs (counted unsolved). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 16.0 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "grok-4-fast",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.58125,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/grok-4-fast-reasoning",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'grok-4-fast-reasoning' (API model in attempt metadata: 'grok-4-fast-reasoning'). grok-4-fast-reasoning endpoint, max_completion_tokens 200000, run 2025-10-07; 8 null attempts. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 232.5 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "grok-4-fast",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.045833,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/grok-4-fast-reasoning",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'grok-4-fast-reasoning' (API model in attempt metadata: 'grok-4-fast-reasoning'). grok-4-fast-reasoning endpoint, run 2025-10-07; 15 null attempts and 4 tasks holding only 1 of 2 test pairs (counted unsolved). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 5.5 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.17,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/qwen3-235b-a22b-instruct-2507",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'qwen3-235b-a22b-instruct-2507' (API model in attempt metadata: 'accounts/fireworks/models/qwen3-235b-a22b-instruct-2507'). Fireworks API, run 2025-07-22. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 68.0 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "qwen3-235b-a22b-instruct-2507",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.008333,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/qwen3-235b-a22b-instruct-2507",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'qwen3-235b-a22b-instruct-2507' (API model in attempt metadata: 'accounts/fireworks/models/qwen3-235b-a22b-instruct-2507'). Fireworks API, run 2025-07-22. ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 1.0 / 120. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.15625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-4-5-2025-02-27",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-4-5-2025-02-27' (API model in attempt metadata: 'gpt-4.5-2025-02-21-alpha'). run 2025-02-26 on the pre-release endpoint 'gpt-4.5-2025-02-21-alpha' (attempt metadata), one day before GPT-4.5's 2025-02-27 release; ARC Prize files it as gpt-4-5-2025-02-27. Identity per the source's folder label (held in pass 8 as a pre-release endpoint). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). Exact-match sum 62.5 / 400. Per-attempt 'correct' flags are null in this folder, so the score was computed by whatmodel.sh by exact grid match of each published attempt 'answer' against the official test outputs (ARC-AGI-1: fchollet/ARC-AGI data/evaluation @399030444e0a; ARC-AGI-2: arcprize/ARC-AGI-2 data/evaluation @2c42f4d6f2ed, the 2025-05-15 version, or @6726759ffd41, the 2025-04-04 version, for runs made before the 2025-05-15 edit; both repos Apache-2.0): pair solved if either of 2 attempts matches, task score = solved pairs/pairs, summed and divided by the task count; null (failed) attempts and missing test pairs count as unsolved. The same grader reproduces every published non-null 'correct' flag in the flagged folders checked (0 disagreements). Held in pass 8 for the grading method; imported under the 2026-10-11 owner decision (re-graded 2026-10-11).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.95625,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-4-high",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-4-high' (API model in attempt metadata: 'galapagos-alpha'). reasoning effort high (highest gpt-5-4 folder; low/medium also published), run 2026-02-27 on the pre-release endpoint 'galapagos-alpha' (attempt metadata), before GPT-5.4's 2026-03-05 release; ARC Prize labels the folder gpt-5-4-high. Identity per the source's label (held in pass 8 as a pre-release endpoint). ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 382.5 / 400); an independent exact-match regrade against the official solutions gives the same value. Held in pass 8 for the pre-release endpoint name; imported under the 2026-10-11 owner decision.",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.758333,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-4-high",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'gpt-5-4-high' (API model in attempt metadata: 'galapagos-alpha'). reasoning effort high (highest gpt-5-4 folder), run 2026-02-27 on the pre-release endpoint 'galapagos-alpha'; ARC Prize labels the folder gpt-5-4-high. Identity per the source's label (held in pass 8 as a pre-release endpoint). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 91 / 120); an independent exact-match regrade against the official solutions gives the same value. Held in pass 8 for the pre-release endpoint name; imported under the 2026-10-11 owner decision.",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gpt-5-4-pro",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.9825,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-4-pro-xhigh",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gpt-5-4-pro-xhigh' (API model in attempt metadata: 'galapagos-pro-alpha'). reasoning effort xhigh, run 2026-03-03/04 on the pre-release endpoint 'galapagos-pro-alpha' (attempt metadata); ARC Prize labels the folder gpt-5-4-pro-xhigh. Identity per the source's label (held in pass 8 as a pre-release endpoint). The v2 run of this folder is incomplete (107/120) and not used. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 393 / 400); an independent exact-match regrade against the official solutions gives the same value. Held in pass 8 for the pre-release endpoint name; imported under the 2026-10-11 owner decision.",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "grok-4-20",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.955,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9/grok-4.20-beta-0309b-reasoning",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'grok-4.20-beta-0309b-reasoning' (API model in attempt metadata: 'grok-4.20-beta-0309b-reasoning'). endpoint 'grok-4.20-beta-0309b-reasoning' (attempt metadata), temperature 0.7, run 2026-03-11; xAI's model page for grok-4.20-0309-reasoning lists the alias 'grok-4.20-beta-0309-reasoning' but not the '0309b' spelling. Identity per the source's Grok 4.20 label (held in pass 8); the separate 'grok-4.20-multi-agent-beta-0309-xhigh' folder is a different product and not used. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 382 / 400); an independent exact-match regrade against the official solutions gives the same value. Held in pass 8 for the pre-release endpoint name; imported under the 2026-10-11 owner decision.",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "grok-4-20",
    "bench": "arcprize-arc-agi-2-public-eval",
    "score": 0.636111,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31/grok-4.20-beta-0309b-reasoning",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v2_public_eval (revision 026789c1c12a4c34580a32e84dcaf5630d7e8f31); folder 'grok-4.20-beta-0309b-reasoning' (API model in attempt metadata: 'grok-4.20-beta-0309b-reasoning'). endpoint 'grok-4.20-beta-0309b-reasoning', run 2026-03-11. Identity per the source's Grok 4.20 label (held in pass 8). ARC-AGI-2 public evaluation set, 120 tasks, 2 attempts per test pair (pass@2). Public evaluation set (not ARC Prize's semi-private leaderboard set). no results.json in the dataset; computed by whatmodel.sh from the per-attempt 'correct' flags in the task files (pair solved if either of 2 attempts is correct; task score = solved pairs/pairs; sum 76.333 / 120); an independent exact-match regrade against the official solutions gives the same value. Held in pass 8 for the pre-release endpoint name; imported under the 2026-10-11 owner decision.",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "gemini-3-flash-preview",
    "bench": "arcprize-arc-agi-1-public-eval",
    "score": 0.31875,
    "source": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-flash-preview-thinking-minimal/results.json",
    "by": "independent",
    "date": "2026-06-04",
    "note": "ARC Prize Foundation model_baseline outputs, HF dataset arcprize/arc_agi_v1_public_eval (revision 3e9c9d1a8402aff82356815c106cecaea65cb7d9); folder 'gemini-3-flash-preview-thinking-minimal' (API model in attempt metadata: 'models/gemini-3-flash-preview'), thinking level minimal, run 2025-12-15. ARC-AGI-1 public evaluation set, 400 tasks, 2 attempts per test pair (pass@2). Published results.json: score 127.5 over total_tasks 400, divided by 400 (the per-attempt flags give the same 127.5). This is the only complete ARC-AGI-1 run of this model in the dataset: the thinking-high (291.5 over 330 tasks), medium (246.5/363) and low (87.0/228) runs are incomplete, so the lowest thinking level is recorded (as with the gpt-5-4-mini v2 'low' row), not the model's highest setting. Held in pass 8 for the 'highest reasoning setting' rule; imported under the 2026-10-11 owner decision. Public evaluation set (not ARC Prize's semi-private leaderboard set).",
    "source_id": "arcprize-model-baseline-outputs"
  },
  {
    "model": "glm-4-5",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.7201,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'GLM-4.5 (FC)' (model link https://huggingface.co/zai-org/GLM-4.5). BFCL V4; 2025-11-03 archive; field Overall Acc = 72.01%. Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-opus-4-1-20250805",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.7121,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'Claude-Opus-4-1-20250805 (FC)' (model link https://www.anthropic.com/news/claude-4). BFCL V4; 2025-11-03 archive; field Overall Acc = 71.21%; the same file also has 'Claude-Opus-4-1-20250805 (Prompt)' = 27.11% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "glm-4-5-air",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.6891,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'GLM-4.5-Air (FC)' (model link https://huggingface.co/zai-org/GLM-4.5-Air). BFCL V4; 2025-11-03 archive; field Overall Acc = 68.91%. Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-5",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.616,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'GPT-5-2025-08-07 (FC)' (model link https://openai.com/index/introducing-gpt-5/). BFCL V4; 2025-11-03 archive; field Overall Acc = 61.60%; the same file also has 'GPT-5-2025-08-07 (Prompt)' = 42.13% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.5231,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'Gemini-2.5-Pro (FC)' (model link https://deepmind.google/technologies/gemini/pro/). BFCL V4; 2025-11-03 archive; field Overall Acc = 52.31%; the same file also has 'Gemini-2.5-Pro (Prompt)' = 53.99% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.5222,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'GPT-4o-2024-11-20 (FC)' (model link https://openai.com/index/hello-gpt-4o/). BFCL V4; 2025-11-03 archive; field Overall Acc = 52.22%; the same file also has 'GPT-4o-2024-11-20 (Prompt)' = 27.73% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "qwq-32b",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.3691,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'QwQ-32B (FC)' (model link https://huggingface.co/Qwen/QwQ-32B). BFCL V4; 2025-11-03 archive; field Overall Acc = 36.91%; the same file also has 'QwQ-32B (Prompt)' = 45.57% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.4206,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'GPT-4o-mini-2024-07-18 (FC)' (model link https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/). BFCL V4; 2025-11-03 archive; field Overall Acc = 42.06%; the same file also has 'GPT-4o-mini-2024-07-18 (Prompt)' = 29.61% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "gemini-2-5-flash-lite-preview-06-17",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.3526,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'Gemini-2.5-Flash-Lite-Preview-06-17 (FC)' (model link https://deepmind.google/technologies/gemini/flash-lite/). BFCL V4; 2025-11-03 archive; field Overall Acc = 35.26%; the same file also has 'Gemini-2.5-Flash-Lite-Preview-06-17 (Prompt)' = 29.19% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "nova-lite",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.3322,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'Amazon-Nova-Lite-v1:0 (FC)' (model link https://aws.amazon.com/cn/ai/generative-ai/nova/). BFCL V4; 2025-11-03 archive; field Overall Acc = 33.22%. Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "phi-4-mini",
    "bench": "bfcl-v4-2025-11-03",
    "score": 0.217,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv",
    "by": "independent",
    "date": "2025-11-03",
    "note": "Published entry 'Phi-4-mini-instruct (FC)' (model link https://huggingface.co/microsoft/Phi-4-mini-instruct). BFCL V4; 2025-11-03 archive; field Overall Acc = 21.70%; the same file also has 'Phi-4-mini-instruct (Prompt)' = 14.43% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-12-16 archive; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-sonnet-4-20250514",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.7029,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'Claude-Sonnet-4-20250514 (FC)' (model link https://www.anthropic.com/news/claude-4). BFCL V4; 2025-08-26 archive; field Overall Acc = 70.29%; the same file also has 'Claude-Sonnet-4-20250514 (Prompt)' = 30.41% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.4897,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'DeepSeek-R1-0528 (FC)' (model link https://api-docs.deepseek.com/news/news250528). BFCL V4; 2025-08-26 archive; field Overall Acc = 48.97%; the same file also has 'DeepSeek-R1-0528 (Prompt)' = 37.38% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-v3-0324",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.452,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'DeepSeek-V3-0324 (FC)' (model link https://api-docs.deepseek.com/news/news250325). BFCL V4; 2025-08-26 archive; field Overall Acc = 45.20%. Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "claude-3-5-haiku-20241022",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.4342,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'claude-3.5-haiku-20241022 (FC)' (model link https://www.anthropic.com/news/3-5-models-and-computer-use). BFCL V4; 2025-08-26 archive; field Overall Acc = 43.42%; the same file also has 'claude-3.5-haiku-20241022 (Prompt)' = 32.66% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "llama-3-1-70b",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.1764,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'Llama-3.1-70B-Instruct (FC)' (model link https://llama.meta.com/llama3). BFCL V4; 2025-08-26 archive; field Overall Acc = 17.64%; the same file also has 'Llama-3.1-70B-Instruct (Prompt)' = 31.55% (not used). Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "glm-4-9b-chat",
    "bench": "bfcl-v4-2025-08-26",
    "score": 0.2119,
    "source": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv",
    "by": "independent",
    "date": "2025-08-26",
    "note": "Published entry 'GLM-4-9b-Chat (FC)' (model link https://huggingface.co/THUDM/glm-4-9b-chat). BFCL V4; 2025-08-26 archive; field Overall Acc = 21.19%. Native function calling (FC) selected when available, otherwise Prompt, as in the bfcl-v4 import. The entry is absent from the 2025-11-03 and 2025-12-16 archives; the archive re-scores entries between snapshots (0 of 87 common entries keep the same Overall Acc from 2025-11-03 to 2025-12-16; 1 of 96 from 2025-08-26 to 2025-11-03), so it is filed under this snapshot's own benchmark id rather than bfcl-v4, and only for models with no later-archive row. Date is archive publication. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "bfcl-result-archive-v4"
  },
  {
    "model": "deepseek-v2-5-1210",
    "bench": "aider-polyglot",
    "score": 0.178,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2024-12-21",
    "note": "Published entry 'DeepSeek Chat V2.5'. Aider polyglot; 225 exercises; pass_rate_2 = 17.8 (after at most two attempts, not pass@1). edit_format=diff; Aider version=0.69.2.dev; evaluator commit=a755079-dirty; command=aider --model deepseek/deepseek-chat; run=2024-12-21-20-56-21--polyglot-deepseek-diff. Identity: on 2024-12-21 the deepseek-chat API alias served DeepSeek-V2.5-1210 (DeepSeek news 'DeepSeek-V2.5-1210 Release 2024/12/10 ... With the release of DeepSeek-V2.5-1210, the V2.5 series comes to an end'; 'Introducing DeepSeek-V3 2024/12/26' came five days after this run), and the label says V2.5, so the only V2.5 checkpoint then served is mapped (the same alias-by-date reasoning as the existing 'DeepSeek Chat V3 (prev)' -> deepseek-v3 row). Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "chatgpt-4o-latest-20250326",
    "bench": "aider-polyglot",
    "score": 0.453,
    "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml",
    "by": "independent",
    "date": "2025-03-29",
    "note": "Published entry 'chatgpt-4o-latest (2025-03-29)'. Aider polyglot; 225 exercises; pass_rate_2 = 45.3 (after at most two attempts, not pass@1). edit_format=diff; Aider version=0.79.3.dev; evaluator commit=0decbad; command=aider --model chatgpt-4o-latest; run=2025-03-29-05-24-55--chatgpt4o-mar28-diff. Identity: the chatgpt-4o-latest alias on 2025-03-29 served the March 2025 GPT-4o update (catalogue identity note quotes OpenAI's release notes: 'Improvements to GPT-4o (March 27, 2025) ... available in ChatGPT and in the API as the newest snapshot of chatgpt-4o-latest'); the next GPT-4o update came later. Held in pass 8 as an alias run; imported under the 2026-10-11 owner decision.",
    "source_id": "aider-polyglot-runs"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "scicode-main-official",
    "score": 0.015,
    "source": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md",
    "by": "independent",
    "date": "2024-07-18",
    "note": "SciCode repository README leaderboard; row 'Gemini 1.5 Pro'; Main Problem Resolve Rate 1.5% (subproblem 21.9% not imported). Maintainers' own evaluation, standard setting without scientist-annotated background; the arXiv v1 paper (2407.13168v1, submitted 2024-07-18) lists the same values. Identity: evaluated before 2024-07-18, when gemini-1.5-pro-001 (GA 2024-05-23 per the Gemini API changelog) was the only stable Gemini 1.5 Pro and gemini-1.5-pro-002 did not exist (released 2024-09-24); the paper does not print an API id (preview ids also existed), so the stable version of the time is mapped, as the GPT-4o row of this table is mapped to gpt-4o-2024-05-13. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "scicode-readme-leaderboard"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "galileo-agent-v1-tsq",
    "score": 0.895,
    "source": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv",
    "by": "independent",
    "date": "2025-07-02",
    "note": "Galileo Agent Leaderboard Space (results_v1.csv, revision 8158b75ae542c2e847ff6d517a0b66caa704b19a); row 'gemini-1.5-flash'; column 'Model Avg' = 0.895; single-turn 0.88, multi-turn 0.91; listed price $0.075 / $0.30 per M tokens (Gemini API list price); 14 tool-calling subsets sampled from BFCL v3, tau-bench, xLAM and ToolACE; TSQ = Galileo ChainPoll LLM-judge (GPT-4o) tool-selection-quality metric, equal-weighted over subsets. Identity: the row first appears in January 2025 (pass-8 history check); the Gemini API changelog (November 21, 2024): 'Updated the gemini-1.5-flash-latest and gemini-1.5-flash model aliases to use gemini-1.5-flash-002', so the unversioned id served -002 at that time. Whether Galileo called the Gemini API or Vertex AI is not stated. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "galileo-agent-leaderboard"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "robloxqa-v2-mc",
    "score": 0.845667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2537/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Qwen 2.5 Coder 32B (Q4_K_M)' (https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct); Q4_K_M GGUF quantisation run locally; no reasoning setting. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen2-5-coder-32b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.245192,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 26+21+4=51/208. Roblox LLM Leaderboard entry 'Qwen 2.5 Coder 32B (Q4_K_M)' (https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct); Q4_K_M GGUF quantisation run locally; no reasoning setting. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "robloxqa-v2-mc",
    "score": 0.821333,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2464/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Qwen 3 4B Instruct 2507 (Q8_0)' (https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507); Q8_0 GGUF quantisation run locally; non-thinking Instruct checkpoint. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-4b-instruct-2507",
    "bench": "luau-leetcode-full-pass",
    "score": 0.153846,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 18+8+6=32/208. Roblox LLM Leaderboard entry 'Qwen 3 4B Instruct 2507 (Q8_0)' (https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507); Q8_0 GGUF quantisation run locally; non-thinking Instruct checkpoint. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "robloxqa-v2-mc",
    "score": 0.907667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2723/3000 correct (noAnswer=7). Roblox LLM Leaderboard entry 'Qwen 3.8 27B (XHigh, Q4_K_XL)' (https://huggingface.co/Qwen/Qwen3.8-27B); Q4_K_XL GGUF quantisation run locally; reasoning XHigh, the higher of the two listed settings (the 'Medium' entry scores 2723/3000 and 42+68+24=134/208). A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-8-27b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.649038,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 40+65+30=135/208. Roblox LLM Leaderboard entry 'Qwen 3.8 27B (XHigh, Q4_K_XL)' (https://huggingface.co/Qwen/Qwen3.8-27B); Q4_K_XL GGUF quantisation run locally; reasoning XHigh, the higher of the two listed settings (the 'Medium' entry scores 2723/3000 and 42+68+24=134/208). A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "robloxqa-v2-mc",
    "score": 0.907,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2721/3000 correct (noAnswer=77). Roblox LLM Leaderboard entry 'Qwen 3.8 Flash Next (XHigh, NVFP4)' (https://huggingface.co/Qwen/Qwen3.8-Flash-Next); NVFP4 quantisation; reasoning XHigh, the higher of the two listed settings (the 'Medium' entry scores 2779/3000 and 45+65+26=136/208). A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-8-flash-next",
    "bench": "luau-leetcode-full-pass",
    "score": 0.764423,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 42+76+41=159/208. Roblox LLM Leaderboard entry 'Qwen 3.8 Flash Next (XHigh, NVFP4)' (https://huggingface.co/Qwen/Qwen3.8-Flash-Next); NVFP4 quantisation; reasoning XHigh, the higher of the two listed settings (the 'Medium' entry scores 2779/3000 and 45+65+26=136/208). A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "robloxqa-v2-mc",
    "score": 0.898667,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "2696/3000 correct (noAnswer=0). Roblox LLM Leaderboard entry 'Muse Glimmer 30B (Q4_K_XL)' (https://huggingface.co/meta-models/Muse-Glimmer-30B); Q4_K_XL GGUF quantisation run locally; reasoning setting not listed. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "muse-glimmer-30b",
    "bench": "luau-leetcode-full-pass",
    "score": 0.682692,
    "source": "https://huggingface.co/spaces/TorpedoSoftware/Roblox-LLM-Leaderboard/blob/676755cd82ed2886048e282f1a6106ddf6e5fea5/data/leaderboard.json",
    "by": "independent",
    "date": "2026-09-30",
    "note": "fullPass 43+70+29=142/208. Roblox LLM Leaderboard entry 'Muse Glimmer 30B (Q4_K_XL)' (https://huggingface.co/meta-models/Muse-Glimmer-30B); Q4_K_XL GGUF quantisation run locally; reasoning setting not listed. A quantised local run of the catalogued checkpoint, not the reference weights: the existing import skipped quantised entries, held in pass 8; imported under the 2026-10-11 owner decision with the quantisation stated here. Date = leaderboard generatedAt 2026-09-30. Space revision 676755cd82ed2886048e282f1a6106ddf6e5fea5.",
    "source_id": "roblox-llm-leaderboard"
  },
  {
    "model": "qwen3-8-2-4t-a95b",
    "bench": "tau3-banking-v101",
    "score": 0.5515,
    "source": "https://github.com/sierra-research/tau2-bench/blob/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699/web/leaderboard/public/submissions/qwen3-8-max_sierra_2026-08-04/submission.json",
    "by": "independent",
    "date": "2026-08-03",
    "note": "τ²-bench leaderboard submission 'qwen3-8-max_sierra_2026-08-04' (sierra-research/tau2-bench @4ce7c03, MIT), submitted by Sierra: model 'Qwen 3.8 Max', domain banking_knowledge, pass^1 = 55.154639175257735% (stored /100, rounded 0.5515). tau2-bench version 1.0.1, user simulator gpt-5.2, reasoning effort xhigh, evaluated 2026-08-03; notes 'Banking_knowledge evaluation with AllTools retrieval, GPT-5.2 low-reasoning user simulation, four trials, and seed 300.' Identity: model_release 2026-07-19 'Qwen3.8-Max-Preview launch'; the trajectory records agent llm 'openrouter/qwen/qwen3.8-max' (pass-8 check). Mapped to the catalogue's Qwen3.8 Max (2.4T-A95B), to which other sources map the undated 'qwen3.8-max' (MathArena config pins HF Qwen/Qwen3.8-2.4T-A95B; Epoch 'qwen3.8-max_xhigh'); the run predates the open-weight release (HF repo created 2026-08-08) and Sierra calls it the Preview, so a pre-release preview served under the same API name. Held in pass 8 for the preview/GA question; imported under the 2026-10-11 owner decision. The later qwen3-8-max-0902 update is a separate catalogue entry.",
    "source_id": "tau2-bench-leaderboard-sierra"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "cv3-eval-en-wer",
    "score": 0.3391,
    "source": "https://github.com/OpenBMB/UltraEval-Audio/blob/bead726925d43f526bed48a4a6a827595169429d/replication/qwen3_tts.md",
    "by": "independent",
    "date": "2026-02-01",
    "note": "UltraEval-Audio replication/qwen3_tts.md, 'CV3-Eval（Zero-shot Voice Clone）' table, column 'en WER/%': printed '33.91±13.06' (eval_cli [11]: dataset cv3_zero_shot_en, model qwen3-tts-0.6b-base). Mean ± spread as printed; the mean is recorded (/100). Scorer per registry/evaluator/cv3.yaml: Whisper large-v3. Identity: row Qwen3-TTS-12Hz-0.6B-Base-official-infer-params (registry qwen3-tts-0.6b-base). The value is far above the 1.7B row (3.77) and the large ± spread suggests some runaway generations, but the source does not flag the run as failed and WER has no random baseline, so it is recorded as a plain measurement (held in pass 8; imported under the 2026-10-11 owner decision).",
    "source_id": "speech-ultraeval-audio-apache"
  },
  {
    "model": "gemini-3-1-flash-tts-preview",
    "bench": "nsanku-ghana7-macro-cer",
    "score": 0.1995,
    "source": "https://huggingface.co/spaces/ghananlpcommunity/nsanku-tts-benchmark/blob/41b828ed6803bd052b1bef94bb6802fecd2edf8c/bundled_data.json",
    "by": "independent",
    "date": "2026-10-01",
    "note": "nsanku TTS Benchmark (GhanaNLP community Space, MIT, revision 41b828e; data 'updated' 2026-10-01), bundled_data.json, entry 'Google/gemini-3.1-flash-tts-preview'; mode noref (the model has no reference-audio mode in this benchmark; the separate 'gemini-3.1-flash-tts-preview-universal' entry adds an africa-g2p grapheme preprocessing step and is not used). Per-language CER as printed: Dangme 0.1659 (199 scored, 1 failed); Dagbani 0.1721 (200 scored, 0 failed); Dagaare 0.1733 (199 scored, 1 failed); Fante 0.2405 (198 scored, 2 failed); Ga 0.2646 (200 scored, 0 failed); Gonja 0.1736 (200 scored, 0 failed); Akuapem Twi 0.2067 (200 scored, 0 failed). Recorded value = unweighted mean of the 7 per-language CERs (0.199529); 200 sentences per language; ASR judges per the file: KhayaAI/khaya-asr-v3 (Dangme, Dagbani, Dagaare, Fante, Gonja) and Qlerqly/griot-nano-1 (Ga, Akuapem Twi). Held in pass 8 (needed a third catalogue model; judge and maker overlap); imported under the 2026-10-11 owner decision.",
    "source_id": "nsanku-tts-benchmark-mit"
  },
  {
    "model": "omnivoice",
    "bench": "nsanku-ghana7-macro-cer",
    "score": 0.1407,
    "source": "https://huggingface.co/spaces/ghananlpcommunity/nsanku-tts-benchmark/blob/41b828ed6803bd052b1bef94bb6802fecd2edf8c/bundled_data.json",
    "by": "independent",
    "date": "2026-10-01",
    "note": "nsanku TTS Benchmark (GhanaNLP community Space, MIT, revision 41b828e; data 'updated' 2026-10-01), bundled_data.json, entry 'k2-fsa/OmniVoice-ref'; mode ref (zero-shot cloning from the benchmark's reference prompt, the model's native cloning mode; the noref/design entries are not used). Per-language CER as printed: Dangme 0.1495 (200 scored, 0 failed); Dagbani 0.0852 (200 scored, 0 failed); Dagaare 0.1127 (200 scored, 0 failed); Fante 0.1338 (200 scored, 0 failed); Ga 0.2483 (189 scored, 11 failed); Gonja 0.0885 (200 scored, 0 failed); Akuapem Twi 0.1669 (200 scored, 0 failed). Recorded value = unweighted mean of the 7 per-language CERs (0.140700); 200 sentences per language; ASR judges per the file: KhayaAI/khaya-asr-v3 (Dangme, Dagbani, Dagaare, Fante, Gonja) and Qlerqly/griot-nano-1 (Ga, Akuapem Twi). Held in pass 8 (needed a third catalogue model; judge and maker overlap); imported under the 2026-10-11 owner decision.",
    "source_id": "nsanku-tts-benchmark-mit"
  },
  {
    "model": "ghana-tts-72k",
    "bench": "nsanku-ghana7-macro-cer",
    "score": 0.1955,
    "source": "https://huggingface.co/spaces/ghananlpcommunity/nsanku-tts-benchmark/blob/41b828ed6803bd052b1bef94bb6802fecd2edf8c/bundled_data.json",
    "by": "lab",
    "date": "2026-10-01",
    "note": "nsanku TTS Benchmark (GhanaNLP community Space, MIT, revision 41b828e; data 'updated' 2026-10-01), bundled_data.json, entry 'ghananlpcommunity/ghana-tts-72k-ref'; mode ref (cloning from the benchmark's reference prompt; the noref entry is not used). GhanaNLP built both this model and the benchmark (and the Khaya ASR judge), so the row is by: lab. Per-language CER as printed: Dangme 0.1255 (200 scored, 0 failed); Dagbani 0.0816 (200 scored, 0 failed); Dagaare 0.0820 (200 scored, 0 failed); Fante 0.0925 (200 scored, 0 failed); Ga 0.8027 (200 scored, 0 failed); Gonja 0.0292 (200 scored, 0 failed); Akuapem Twi 0.1552 (200 scored, 0 failed). Recorded value = unweighted mean of the 7 per-language CERs (0.195529); 200 sentences per language; ASR judges per the file: KhayaAI/khaya-asr-v3 (Dangme, Dagbani, Dagaare, Fante, Gonja) and Qlerqly/griot-nano-1 (Ga, Akuapem Twi). Held in pass 8 (needed a third catalogue model; judge and maker overlap); imported under the 2026-10-11 owner decision.",
    "source_id": "nsanku-tts-benchmark-mit"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.0168,
    "source": "https://github.com/sgl-project/sglang-omni/blob/9641219b320597393d915afb8d03352c629d94a5/docs/cookbook/moss_tts.md",
    "by": "independent",
    "date": "2026-10-09",
    "note": "SGLang-Omni cookbook docs/cookbook/moss_tts.md (sgl-project/sglang-omni @9641219, Apache-2.0), 'Benchmark Results': 'Seed-TTS-Eval full set (EN = 1088, ZH = 2020) on 1× H200, concurrency 8, --token-count auto. WER is scored with HF Whisper-large-v3 (EN) / FunASR paraformer-zh (ZH)'; table row 'EN', 'WER (corpus)' = 1.68% (raw corpus micro-average over all samples; 'WER (excl. >50%)' 1.32% with 4 outliers not used). Model OpenMOSS-Team/MOSS-TTS-v1.5, which 'clones from each prompt (--ref-format references)'. Serving-validation run by the SGLang-Omni team (numbers 'tabulated in benchmarks/eval/benchmark_tts_seedtts.py (source: PR #609)'). Held in pass 8 (serving-validation protocols differ per cookbook); imported under the 2026-10-11 owner decision where the cookbook's protocol matches an existing id.",
    "source_id": "speech-sglang-omni-cookbooks-apache"
  },
  {
    "model": "moss-tts-v1-5",
    "bench": "seed-tts-zh-paraformer-cer",
    "score": 0.0136,
    "source": "https://github.com/sgl-project/sglang-omni/blob/9641219b320597393d915afb8d03352c629d94a5/docs/cookbook/moss_tts.md",
    "by": "independent",
    "date": "2026-10-09",
    "note": "SGLang-Omni cookbook docs/cookbook/moss_tts.md (@9641219, Apache-2.0), 'Benchmark Results', row 'ZH', 'WER (corpus)' = 1.36% (Chinese scored with FunASR paraformer-zh, so a character error rate; raw corpus over 2,020 samples; 1.27% excluding 2 outliers not used). Same run and setup as the EN row: MOSS-TTS-v1.5, cloning from each prompt, concurrency 8, --token-count auto. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "speech-sglang-omni-cookbooks-apache"
  },
  {
    "model": "qwen3-tts-12hz-0-6b-base",
    "bench": "seed-tts-en-whisper-v3-wer",
    "score": 0.1829,
    "source": "https://github.com/sgl-project/sglang-omni/blob/9641219b320597393d915afb8d03352c629d94a5/docs/cookbook/qwen3_tts.md",
    "by": "independent",
    "date": "2026-10-09",
    "note": "SGLang-Omni cookbook docs/cookbook/qwen3_tts.md (@9641219, Apache-2.0), 'Benchmark Results' > '0.6B Base': 'Qwen3-TTS-12Hz-0.6B-Base on Seed-TTS EN (1088 utterances, reference voice cloning from each prompt), concurrency 16, WER scored with HF Whisper-large-v3'; 'WER (corpus micro-avg, raw)' = 18.29% (recorded, as for the MOSS-TTS row: every sample included). The cookbook explains: 'Two utterances (0.2%) ran away into a repetition loop ... which alone lifts the raw micro-average to 18.29%; excluding those, corpus WER is 1.07%' (per-sample median 0.00%). The source does not call the run failed. Held in pass 8; imported under the 2026-10-11 owner decision.",
    "source_id": "speech-sglang-omni-cookbooks-apache"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "fleurs-en-wer",
    "score": 0.0409,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-en', column 'Parakeet' = 4.09; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Parakeet: Nvidia Parakeet-TDT-0.6B-v2' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.0399,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-en', column 'Whisper' = 3.99; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Whisper: OpenAI Whisper Large-v3' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "sensevoice-small",
    "bench": "fleurs-en-wer",
    "score": 0.0684,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-en', column 'SenseVoice' = 6.84; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'SenseVoice: SenseVoice-small' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "fleurs-en-wer",
    "score": 0.0493,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-en', column 'FunASR' = 4.93; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'FunASR: FunASR-Nano (the zh/en Fun-ASR-Nano-2512 release; the paper says FunASR-Nano is 'restricted to one or two languages', so not the MLT variant)' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "sensevoice-small",
    "bench": "fleurs-zh-err",
    "score": 0.0556,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-zh', column 'SenseVoice' = 5.56; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'SenseVoice: SenseVoice-small' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "fleurs-zh-err",
    "score": 0.07,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Fleurs-zh', column 'FunASR' = 7.00; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'FunASR: FunASR-Nano (the zh/en Fun-ASR-Nano-2512 release; the paper says FunASR-Nano is 'restricted to one or two languages', so not the MLT variant)' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0149,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-clean', column 'Parakeet' = 1.49; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Parakeet: Nvidia Parakeet-TDT-0.6B-v2' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0198,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-clean', column 'Whisper' = 1.98; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Whisper: OpenAI Whisper Large-v3' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "sensevoice-small",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0278,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-clean', column 'SenseVoice' = 2.78; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'SenseVoice: SenseVoice-small' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0158,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-clean', column 'FunASR' = 1.58; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'FunASR: FunASR-Nano (the zh/en Fun-ASR-Nano-2512 release; the paper says FunASR-Nano is 'restricted to one or two languages', so not the MLT variant)' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "librispeech-test-other-wer",
    "score": 0.0313,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-other', column 'Parakeet' = 3.13; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Parakeet: Nvidia Parakeet-TDT-0.6B-v2' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.036,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-other', column 'Whisper' = 3.60; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Whisper: OpenAI Whisper Large-v3' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "sensevoice-small",
    "bench": "librispeech-test-other-wer",
    "score": 0.0681,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-other', column 'SenseVoice' = 6.81; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'SenseVoice: SenseVoice-small' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "librispeech-test-other-wer",
    "score": 0.0401,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'Libri-other', column 'FunASR' = 4.01; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'FunASR: FunASR-Nano (the zh/en Fun-ASR-Nano-2512 release; the paper says FunASR-Nano is 'restricted to one or two languages', so not the MLT variant)' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "parakeet-tdt-0-6b-v2",
    "bench": "voxpopuli-en-wer",
    "score": 0.0526,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'VoxPopuli', column 'Parakeet' = 5.26; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Parakeet: Nvidia Parakeet-TDT-0.6B-v2' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "voxpopuli-en-wer",
    "score": 0.0719,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'VoxPopuli', column 'Whisper' = 7.19; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'Whisper: OpenAI Whisper Large-v3' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "sensevoice-small",
    "bench": "voxpopuli-en-wer",
    "score": 0.0863,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'VoxPopuli', column 'SenseVoice' = 8.63; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'SenseVoice: SenseVoice-small' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "fun-asr-nano-2512",
    "bench": "voxpopuli-en-wer",
    "score": 0.0646,
    "source": "https://huggingface.co/microsoft/VibeVoice-ASR-BitNet/blob/66e78021ab8f5f06133d1ab421ba4d348bda97c9/README.md",
    "by": "independent",
    "date": "2026-07-24",
    "note": "VibeVoice-ASR-BitNet model card (Microsoft Research, MIT, pinned 66e7802), 'Accuracy (WER%)' table, row 'VoxPopuli', column 'FunASR' = 6.46; divided by 100. Same table as arXiv:2607.21075v2 Table 4 (CC BY 4.0), whose caption names the comparator: 'FunASR: FunASR-Nano (the zh/en Fun-ASR-Nano-2512 release; the paper says FunASR-Nano is 'restricted to one or two languages', so not the MLT variant)' and states '“—” indicates unsupported language or abnormally high error rate', i.e. the authors ran the comparators themselves (values also differ from the makers' own figures, e.g. Parakeet v2 LibriSpeech clean/other 1.49/3.13 here vs 1.69/3.19 on NVIDIA's card). Held in pass 8 for the missing explicit statement; imported under the 2026-10-11 owner decision. Decoding/normalisation follow the authors' harness (not stated).",
    "source_id": "card3-microsoft-vibevoice-asr-bitnet"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "fleurs-en-wer",
    "score": 0.058,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "independent",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | FLEURS', column 'Seamless M4T large v2' = 5.8% WER; divided by 100. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Comparator column treated as the GigaAM team's own run: the same column carries Russian/Kazakh/Kyrgyz/Uzbek 'Internal' in-the-wild test-set cells that only the authors can have measured, under the card's own filtering. Digit-containing references and >30 s utterances excluded, so not the full test set. Held in pass 8 (no explicit statement); imported under the 2026-10-11 owner decision. The 'Omnilingual 1B (LLM)' column is not used: v1 or v2 not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.162,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "independent",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | CV', column 'Seamless M4T large v2' = 16.2% WER; divided by 100. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Comparator column treated as the GigaAM team's own run: the same column carries Russian/Kazakh/Kyrgyz/Uzbek 'Internal' in-the-wild test-set cells that only the authors can have measured, under the card's own filtering. Digit-containing references and >30 s utterances excluded, so not the full test set. Held in pass 8 (no explicit statement); imported under the 2026-10-11 owner decision. The 'Omnilingual 1B (LLM)' column is not used: v1 or v2 not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-en-wer",
    "score": 0.039,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "independent",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | FLEURS', column 'Whisper large v3' = 3.9% WER; divided by 100. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Comparator column treated as the GigaAM team's own run: the same column carries Russian/Kazakh/Kyrgyz/Uzbek 'Internal' in-the-wild test-set cells that only the authors can have measured, under the card's own filtering. Digit-containing references and >30 s utterances excluded, so not the full test set. Held in pass 8 (no explicit statement); imported under the 2026-10-11 owner decision. The 'Omnilingual 1B (LLM)' column is not used: v1 or v2 not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "whisper-large-v3",
    "bench": "commonvoice-en-wer-version-unspecified",
    "score": 0.2,
    "source": "https://huggingface.co/ai-sage/GigaAM-Multilingual/blob/2f8a57144e6ec3adfd32fe0484d9ea9913305bc8/README.md",
    "by": "independent",
    "date": "2026-07-14",
    "note": "GigaAM Multilingual card @2f8a571 (MIT), 'Model Performance' table, row 'English | CV', column 'Whisper large v3' = 20.0% WER; divided by 100. Card 'Model Performance': 'Word Error Rate (%) on Common Voice (CV), FLEURS, and internal in-the-wild test sets. Utterances longer than 30 s and references containing digits are excluded; references/hypotheses are normalized (lowercasing, punctuation removal, numerals→words); greedy decoding.' Comparator column treated as the GigaAM team's own run: the same column carries Russian/Kazakh/Kyrgyz/Uzbek 'Internal' in-the-wild test-set cells that only the authors can have measured, under the card's own filtering. Digit-containing references and >30 s utterances excluded, so not the full test set. Held in pass 8 (no explicit statement); imported under the 2026-10-11 owner decision. The 'Omnilingual 1B (LLM)' column is not used: v1 or v2 not stated.",
    "source_id": "card3-ai-sage-gigaam-multilingual"
  },
  {
    "model": "step1x-edit-v1-0",
    "bench": "lmarena-image-edit",
    "score": 998.3,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/image_edit/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-06",
    "note": "image_edit, category overall, split latest; exact entry 'step1x-edit'; 155,418 votes; published 95% interval 993.8–1002.7; leaderboard_publish_date 2026-10-06. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Identity: Arena's leaderboard table labels the key 'Step1X-Edit' (StepFun, Apache 2.0) linking stepfun-ai/Step1X-Edit; the entry was on the first image_edit publish (2025-06-24, 2,373 votes), when only v1.0 existed (v1.1 was added to the same repo on 2025-07-09), and Arena normally adds a new entry for a new version, so the v1.0 mapping is the better one (not confirmed that Arena never swapped checkpoints). Held in pass 8/9; imported under the 2026-10-11 owner decision.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "minimax-m2-1",
    "bench": "lmarena-text",
    "score": 1384.6,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/text_style_control/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "text_style_control, category overall, split latest; exact entry 'minimax-m2.1-preview'; 17,248 votes; published 95% interval 1379.5–1389.8; leaderboard_publish_date 2026-10-08. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Identity: Arena has never published an unsuffixed 'minimax-m2.1'; the preview entry first appears on 2025-12-30 (text), after MiniMax published MiniMaxAI/MiniMax-M2.1 (HF repo created 2025-12-20), so it is mapped to the released checkpoint (preview label; held in pass 9, imported under the 2026-10-11 owner decision).",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "minimax-m2-1",
    "bench": "lmarena-webdev",
    "score": 1384.4,
    "source": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/088c607919d1d0d3385409070657f4623e414872/webdev/latest-00000-of-00001.parquet",
    "by": "independent",
    "date": "2026-10-08",
    "note": "webdev, category overall, split latest; exact entry 'minimax-m2.1-preview'; 7,645 votes; published 95% interval 1374.9–1393.9; leaderboard_publish_date 2026-10-08. Pinned to dataset revision 088c6079 (re-read 2026-10-11). Identity: as for the lmarena-text row (preview entry first published 2025-12-29 on webdev, after the 2025-12-20 MiniMax-M2.1 release; no unsuffixed entry exists). Held in pass 9; imported under the 2026-10-11 owner decision.",
    "source_id": "lmarena-dataset"
  },
  {
    "model": "qwen-agentworld-35b-a3b",
    "bench": "agentworldbench-overall",
    "score": 0.5639,
    "source": "https://huggingface.co/Qwen/Qwen-AgentWorld-35B-A3B/blob/60d2b0434a53d2e62a7c00a489586815d94ebffb/README.md",
    "by": "lab",
    "date": "2026-06-25",
    "note": "Qwen-AgentWorld-35B-A3B model card (Apache-2.0, pinned 60d2b04), 'AgentWorldBench (Open-Ended Evaluation)' table, row 'Qwen-AgentWorld-35B-A3B', column 'Overall' = 56.39; divided by 100. Benchmark introduced by the same authors (their own runs). Reasoning settings of the API models are not stated. DeepSeek-V4-Pro row not used (snapshot ambiguous); Qwen-AgentWorld-397B-A17B row not used (no released weights). Re-read 2026-10-11. Held at the 2026-10-10 merge (only result is a specialist test built for this model); imported under the 2026-10-11 owner decision.",
    "source_id": "card3-qwen-qwen-agentworld-35b-a3b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "mmlu",
    "score": 0.4493,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'MMLU' = 44.93; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "gpqa-unspecified",
    "score": 0.2214,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'GPQA' = 22.14; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "ifeval-unspecified",
    "score": 0.6424,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'IFEval' = 64.24; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "ifbench-unspecified",
    "score": 0.1975,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'IFBench' = 19.75; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "gsm8k",
    "score": 0.3647,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'GSM8K' = 36.47; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "mgsm",
    "score": 0.4128,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'MGSM' = 41.28; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-0-6b",
    "bench": "mmmlu",
    "score": 0.3084,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-0.6B', column 'MMMLU' = 30.84; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmlu",
    "score": 0.5911,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'MMLU' = 59.11; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gpqa-unspecified",
    "score": 0.2772,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'GPQA' = 27.72; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifeval-unspecified",
    "score": 0.7398,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'IFEval' = 73.98; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "ifbench-unspecified",
    "score": 0.2127,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'IFBench' = 21.27; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "gsm8k",
    "score": 0.514,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'GSM8K' = 51.4; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mgsm",
    "score": 0.6656,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'MGSM' = 66.56; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "qwen3-1-7b",
    "bench": "mmmlu",
    "score": 0.4651,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Qwen3-1.7B', column 'MMMLU' = 46.51; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmlu",
    "score": 0.466,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'MMLU' = 46.6; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gpqa-unspecified",
    "score": 0.2884,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'GPQA' = 28.84; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifeval-unspecified",
    "score": 0.5239,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'IFEval' = 52.39; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "ifbench-unspecified",
    "score": 0.1686,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'IFBench' = 16.86; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "gsm8k",
    "score": 0.3571,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'GSM8K' = 35.71; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mgsm",
    "score": 0.2912,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'MGSM' = 29.12; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "llama-3-2-1b",
    "bench": "mmmlu",
    "score": 0.3815,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'Llama-3.2-1B-Instruct', column 'MMMLU' = 38.15; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmlu",
    "score": 0.4008,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'MMLU' = 40.08; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gpqa-unspecified",
    "score": 0.2107,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'GPQA' = 21.07; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifeval-unspecified",
    "score": 0.629,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'IFEval' = 62.9; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "ifbench-unspecified",
    "score": 0.1772,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'IFBench' = 17.72; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "gsm8k",
    "score": 0.5959,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'GSM8K' = 59.59; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "mgsm",
    "score": 0.436,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'MGSM' = 43.6; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "gemma-3-1b",
    "bench": "mmmlu",
    "score": 0.3443,
    "source": "https://huggingface.co/LiquidAI/LFM2-1.2B/blob/40f3da0d0164913923aee9462c23077868b816a3/README.md",
    "by": "independent",
    "date": "2026-08-05",
    "note": "LFM2-1.2B model card (LFM Open License v1.0, pinned 40f3da0), 'Automated benchmarks' table, row 'gemma-3-1b-it', column 'MMMLU' = 34.43; divided by 100 (bold/italic markup removed). Comparator row, Liquid AI's own run: the same values are printed in the LFM2 technical report (arXiv 2511.23404v1, CC BY 4.0) Table 6, whose caption says 'All results are obtained using our internal evaluation harness, and may differ from other reported scores'. Recorded in this card's run (where the LFM2-350M/700M/1.2B rows already sit) rather than the report's, so each measurement appears once. Held in pass 9 (card gives no statement); imported under the 2026-10-11 owner decision. Qwen3 thinking toggle not stated. GPQA variant and MGSM/MMMLU language sets not stated.",
    "source_id": "card3-liquidai-lfm2-1-2b"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "mmlu",
    "score": 0.6442,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU', column 'LFM2-2.6B' = 64.42. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "mmlu-pro",
    "score": 0.2596,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMLU-Pro', column 'LFM2-2.6B' = 25.96. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "gpqa-diamond",
    "score": 0.2657,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GPQA', column 'LFM2-2.6B' = 26.57. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "ifeval-mean-four",
    "score": 0.7956,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFEval', column 'LFM2-2.6B' = 79.56. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "ifbench-unspecified",
    "score": 0.2219,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'IFBench', column 'LFM2-2.6B' = 22.19. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "multi-if-three-turn-average",
    "score": 0.6026,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'Multi-IF', column 'LFM2-2.6B' = 60.26. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "gsm8k-loose-substring",
    "score": 0.8241,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSM8K', column 'LFM2-2.6B' = 82.41. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "gsm-plus",
    "score": 0.6075,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'GSMPlus', column 'LFM2-2.6B' = 60.75. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "math-500",
    "score": 0.636,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MATH 500', column 'LFM2-2.6B' = 63.60. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "mmmlu-liquid-seven-language",
    "score": 0.5539,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MMMLU', column 'LFM2-2.6B' = 55.39. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "lfm2-2-6b",
    "bench": "mgsm-liquid-six-language-loose",
    "score": 0.7432,
    "source": "https://arxiv.org/html/2511.23404v1",
    "by": "lab",
    "date": "2026-09-25",
    "note": "LFM2 technical report (Liquid AI, arXiv 2511.23404v1, CC BY 4.0), Table 7, row 'MGSM', column 'LFM2-2.6B' = 74.32. Caption: 'All results are obtained using our internal evaluation harness, and may differ from other reported scores.' Same row, harness and bench id as the existing Table 7 cells of this run (LFM2-8B-A1B, Llama-3.2-3B, SmolLM3-3B, Gemma-3-4B, Qwen3-4B); MATH Lvl 5 not used, as for the other columns. Checkpoint: LiquidAI/LFM2-2.6B (Liquid AI own model; catalogue lfm2-2-6b, added after the pass-7 extraction of this table). Re-read 2026-10-11. Added in pass 12 for the held LFM2-2.6B card item: the card's comparator cells equal this table's (already recorded from the report), so the card itself is not used, to avoid recording the same measurements twice.",
    "source_id": "report-2511.23404v1"
  },
  {
    "model": "yi-large",
    "bench": "cruxeval-zeroeval",
    "score": 0.6025,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large' (undated; LMArena's leaderboard table lists 'yi-large' as 'Yi-Large', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8117,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large' (undated; LMArena's leaderboard table lists 'yi-large' as 'Yi-Large', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large",
    "bench": "zebralogic-zeroeval",
    "score": 0.188,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.80. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large' (undated; LMArena's leaderboard table lists 'yi-large' as 'Yi-Large', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large",
    "bench": "gsm8k-zeroeval",
    "score": 0.8006,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large' (undated; LMArena's leaderboard table lists 'yi-large' as 'Yi-Large', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large-preview",
    "bench": "cruxeval-zeroeval",
    "score": 0.6038,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large-preview'. ZeroEval crux; zero-shot reasoning + JSON answer; Mode=greedy; 800 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large-preview' (undated; LMArena's leaderboard table lists 'yi-large-preview' as 'Yi-Large-preview', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large-preview",
    "bench": "mmlu-redux-zeroeval",
    "score": 0.8215,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large-preview'. ZeroEval mmlu-redux; zero-shot reasoning + JSON answer; Mode=greedy; 2778 evaluated items; field=Acc; no-answer percentage=0.14. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large-preview' (undated; LMArena's leaderboard table lists 'yi-large-preview' as 'Yi-Large-preview', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large-preview",
    "bench": "zebralogic-zeroeval",
    "score": 0.189,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large-preview'. ZeroEval zebra-grid; zero-shot reasoning + JSON answer; Mode=greedy; 1000 evaluated items; field=Puzzle Acc; no-answer percentage=1.40. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large-preview' (undated; LMArena's leaderboard table lists 'yi-large-preview' as 'Yi-Large-preview', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large-preview",
    "bench": "gsm8k-zeroeval",
    "score": 0.8264,
    "source": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Published entry 'yi-large-preview'. ZeroEval gsm; zero-shot reasoning + JSON answer; Mode=greedy; 1319 evaluated items; field=Acc; no-answer percentage=0.00. One published single-sample run; no voting, retries, sampling or follow-up variants imported. The repository describes temperature=0/top_p=1 defaults; actual per-run token limit is not in the summary and is not asserted. Snapshot committed 2025-04-14; inference dates unspecified. Identity: 01.AI's API model id 'yi-large-preview' (undated; LMArena's leaderboard table lists 'yi-large-preview' as 'Yi-Large-preview', 01 AI, Proprietary); comparison-only catalogue entry added in pass 12 (held in pass 9 as an undated vendor id; imported under the 2026-10-11 owner decision).",
    "source_id": "zeroeval-published-results"
  },
  {
    "model": "yi-large",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 48.9273,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'yi-large': WB_score.task_macro = 48.92726960200772 (stored rounded to 4 dp). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Identity: 01.AI API model id 'yi-large' (undated); comparison-only entry added in pass 12 (held in pass 9; imported under the 2026-10-11 owner decision).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "yi-large-preview",
    "bench": "wildbench-v2-wb-score-macro",
    "score": 55.2946,
    "source": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json",
    "by": "independent",
    "date": "2024-08-01",
    "note": "WildBench v2 leaderboard data (allenai/WildBench @d6b8dca, leaderboard/data_dir/all_stat_wildbench.-1.json), entry 'yi-large-preview': WB_score.task_macro = 55.294625232024785 (stored rounded to 4 dp). GPT-4o-judged checklist scores on 1,024 real-user tasks, rescaled per WildBench as (score-5)x2x10 and macro-averaged over the five task categories; stored raw (signed, not a fraction). Identity: 01.AI API model id 'yi-large-preview' (undated); comparison-only entry added in pass 12 (held in pass 9; imported under the 2026-10-11 owner decision).",
    "source_id": "wildbench-leaderboard-data"
  },
  {
    "model": "yi-large",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.637,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'yi-large' score 63.7 (95% CI -2.6, 2.4; average #tokens 626); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). The style-control table's yi-large / yi-large-preview rows are not used. Identity: 01.AI API model id 'yi-large' (undated); comparison-only entry added in pass 12 (held in pass 9; imported under the 2026-10-11 owner decision).",
    "source_id": "arena-hard-auto-readme-v0-1"
  },
  {
    "model": "yi-large-preview",
    "bench": "alpacaeval2-lc",
    "score": 0.5189,
    "source": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv",
    "by": "independent",
    "date": "2025-08-09",
    "note": "AlpacaEval 2.0 official leaderboard (tatsu-lab/alpaca_eval @cd543a1, Apache-2.0), weighted_alpaca_eval_gpt4_turbo_leaderboard.csv, row 'yi-large-preview' (mode 'verified'), length_controlled_winrate = 51.894415134099546 (stored 0.5189); raw win rate 57.47, 805 instructions judged by weighted_alpaca_eval_gpt4_turbo against gpt4_1106_preview. Model config models_configs/yi-large-preview/configs.yaml: openai_completions, model_name 'yi-large-preview', temperature 0.6, top_p 0.9, max_tokens 4000 (added 2024-05-08, PR #304 'Add Yi-Large Preview to AlpacaEval'). Comparison-only entry added in pass 12 (held in pass 9; imported under the 2026-10-11 owner decision).",
    "source_id": "alpaca-eval-leaderboard-apache"
  },
  {
    "model": "qwen2-57b-a14b-instruct",
    "bench": "bigcodebench-complete",
    "score": 0.465,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'Qwen2-Instruct' / checkpoint '57B-A14B', column 'Greedy Decoding · Original' = 0.465 (calibrated 0.468). Original (uncalibrated) greedy pass@1 used for every row so base and instruct models share one metric. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: the row sits under 'Qwen2-Instruct', and Table 7 lists instruction-tuned models only, so it is Qwen2-57B-A14B-Instruct, although Table 4 ('Models for Evaluations') links the base repo huggingface.co/Qwen/Qwen2-57B-A14B (the instruct reading is clearly the better one; held in pass 9 for that conflict, imported under the 2026-10-11 owner decision). Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "qwen2-57b-a14b-instruct",
    "bench": "bigcodebench-instruct",
    "score": 0.357,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "lab",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'Qwen2-Instruct' / checkpoint '57B-A14B', column 'Greedy Decoding · Original' = 0.357 (calibrated 0.361). Original greedy pass@1 used consistently. Authors' own runs: §4 'we extensively evaluate 60 state-of-the-art LLMs on BigCodeBench-Complete and 35 instruction-tuned LLMs on BigCodeBench-Instruct ... we report Pass@1 with greedy decoding for the main experiments in the zero-shot setting'. Full 1,140-task set. Identity: the row sits under 'Qwen2-Instruct', and Table 7 lists instruction-tuned models only, so it is Qwen2-57B-A14B-Instruct, although Table 4 ('Models for Evaluations') links the base repo huggingface.co/Qwen/Qwen2-57B-A14B (the instruct reading is clearly the better one; held in pass 9 for that conflict, imported under the 2026-10-11 owner decision). Qwen-family row marked lab: co-author Binyuan Hui is a Qwen team author (Qwen technical report, Bai et al. 2023).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "claude-instant-1-2",
    "bench": "humaneval",
    "score": 0.573,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-instant-1 (Mar 2024)' (link https://www.anthropic.com/news/releasing-claude-instant-1-2; chat/instruction prompt, 'prompted': true); pass@1 humaneval = 57.3 (greedy decoding, per the leaderboard notes), divided by 100. Original HumanEval tests. Per-entry evaluation date not published beyond the '(Mar 2024)' label; date is the data file's last change. Identity: the entry's own link is Anthropic's 'Releasing Claude Instant 1.2' post, and claude-instant-1.2 was the Claude Instant version served in March 2024, so the alias run maps to catalogue claude-instant-1-2. Held in pass 9 as an alias entry; imported under the 2026-10-11 owner decision. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-instant-1-2",
    "bench": "humaneval-plus",
    "score": 0.506,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-instant-1 (Mar 2024)' (link https://www.anthropic.com/news/releasing-claude-instant-1-2; chat/instruction prompt, 'prompted': true); pass@1 humaneval+ = 50.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). Per-entry evaluation date not published beyond the '(Mar 2024)' label; date is the data file's last change. Identity: the entry's own link is Anthropic's 'Releasing Claude Instant 1.2' post, and claude-instant-1.2 was the Claude Instant version served in March 2024, so the alias run maps to catalogue claude-instant-1-2. Held in pass 9 as an alias entry; imported under the 2026-10-11 owner decision. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-2-0",
    "bench": "humaneval",
    "score": 0.695,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-2 (Mar 2024)' (link https://www.anthropic.com/news/claude-2; chat/instruction prompt, 'prompted': true); pass@1 humaneval = 69.5 (greedy decoding, per the leaderboard notes), divided by 100. Original HumanEval tests. Per-entry evaluation date not published beyond the '(Mar 2024)' label; date is the data file's last change. Identity: the entry's own link is Anthropic's Claude 2 announcement (anthropic.com/news/claude-2), not the Claude 2.1 post, so the 'claude-2' alias run maps to catalogue claude-2-0 (the Claude 2 model that announcement introduced); Claude 2.1 (Nov 2023) had its own id claude-2.1. Held in pass 9 as an alias entry; imported under the 2026-10-11 owner decision. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "claude-2-0",
    "bench": "humaneval-plus",
    "score": 0.616,
    "source": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "EvalPlus leaderboard data file results.json @8741845 (last changed 2024-12-26), entry 'claude-2 (Mar 2024)' (link https://www.anthropic.com/news/claude-2; chat/instruction prompt, 'prompted': true); pass@1 humaneval+ = 61.6 (greedy decoding, per the leaderboard notes), divided by 100. HumanEval+ (EvalPlus expanded tests, v0.1.10). Per-entry evaluation date not published beyond the '(Mar 2024)' label; date is the data file's last change. Identity: the entry's own link is Anthropic's Claude 2 announcement (anthropic.com/news/claude-2), not the Claude 2.1 post, so the 'claude-2' alias run maps to catalogue claude-2-0 (the Claude 2 model that announcement introduced); Claude 2.1 (Nov 2023) had its own id claude-2.1. Held in pass 9 as an alias entry; imported under the 2026-10-11 owner decision. Apache-2.0 (repository LICENSE).",
    "source_id": "evalplus-published-runs"
  },
  {
    "model": "glm-4-0116",
    "bench": "mmlu",
    "score": 0.815,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'MMLU' = 81.5; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "gsm8k",
    "score": 0.876,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'GSM8K' = 87.6; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "math-subset-unspecified",
    "score": 0.479,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'MATH' = 47.9; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "bbh",
    "score": 0.823,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'BBH' = 82.3; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "gpqa-unspecified",
    "score": 0.357,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'GPQA' = 35.7; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "humaneval",
    "score": 0.72,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0116)', column 'HumanEval' = 72.0; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "mmlu",
    "score": 0.833,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'MMLU' = 83.3; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "gsm8k",
    "score": 0.933,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'GSM8K' = 93.3; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "math-subset-unspecified",
    "score": 0.613,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'MATH' = 61.3; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "bbh",
    "score": 0.847,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'BBH' = 84.7; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "gpqa-unspecified",
    "score": 0.399,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'GPQA' = 39.9; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0520",
    "bench": "humaneval",
    "score": 0.785,
    "source": "https://arxiv.org/html/2406.12793v2",
    "by": "lab",
    "date": "2024-07-30",
    "note": "ChatGLM family report (Team GLM / Zhipu AI, arXiv 2406.12793v2, CC BY 4.0), Table 2 'GLM-4 performance on academic benchmarks', row 'GLM-4 (0520)', column 'HumanEval' = 78.5; divided by 100. Zhipu's own model, reported by its makers (by: lab); shots/prompting not stated in the table; GLM-4 'deployed with BFloat16 precision' (§3). Only the GLM rows are used (the GPT-4/Claude/Gemini baseline rows are not the authors' runs: e.g. GPT-4 (0314) MMLU 86.4 / HumanEval 67.0 are OpenAI's published figures). Added in pass 12 with the held glm-4-0116 item so the new model and its 0520 successor share this lab run.",
    "source_id": "paper-2406.12793v2-chatglm"
  },
  {
    "model": "glm-4-0116",
    "bench": "arena-hard-v0-1-gpt4-1106",
    "score": 0.557,
    "source": "https://github.com/lmarena/arena-hard-auto/blob/3071ef95d913bb5cd632641db336a3b6c7fe4bdd/README.md",
    "by": "independent",
    "date": "2024-11-14",
    "note": "Arena-Hard-Auto README at commit 3071ef95d9, section 'Leaderboard' ('The following leaderboard has no style control', Updated: 11/14 = 2024-11-14), row 'glm-4-0116' score 55.7 (95% CI -2.4, 2.3; average #tokens 622); Arena-Hard-Auto v0.1, 500 prompts, win rate against baseline gpt-4-0314 judged by GPT-4-Turbo (gpt-4-1106-preview). Identity: LMSYS key 'glm-4-0116' (LMArena's leaderboard table: 'GLM-4-0116', Zhipu AI, Proprietary, https://open.bigmodel.cn/) = GLM-4 (0116), which Zhipu's ChatGLM report (arXiv 2406.12793v2) says 'Since January 16, 2024 ... has been made available through the GLM-4 API at https://bigmodel.cn'. Held in pass 9 (no Zhipu page for the dated id); imported under the 2026-10-11 owner decision.",
    "source_id": "arena-hard-auto-readme-v0-1"
  },
  {
    "model": "yehia-7b-reasoning-preview",
    "bench": "aragen-v3-3c3h",
    "score": 0.45,
    "source": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json",
    "by": "independent",
    "date": "2025-09-30",
    "note": "Inception AraGen-v3 leaderboard entry 'JasperV13/Yehia-7B-Reasoning-preview' (Revision main, Precision bfloat16); 3C3H Score 0.45 (95% CI 0.403717-0.497167, n=324); judge claude-sonnet-4-20250514; 324/324 entries answered; run 20250930_171547; RM_Thinking_Traces=None. The same repo also has a float16 run (20250930_151434) scoring 0.4525 (CI 0.405692-0.499484); the bfloat16 run is used because it matches the checkpoint's native dtype (config.json torch_dtype bfloat16). Held in pass 9 (two runs, no rule to pick one); imported under the 2026-10-11 owner decision.",
    "source_id": "inception-leaderboards-space"
  },
  {
    "model": "hivemind-32b-preview",
    "bench": "eqbench3-elo",
    "score": 1350.99,
    "source": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz",
    "by": "independent",
    "date": "2026-05-10",
    "note": "EQ-Bench 3 canonical leaderboard data (data/canonical_leaderboard_elo_results.json.gz, metadata last_updated 2026-05-10T03:34:49.858926+00:00); entry 'HiveLabsAI/hivemind-32b-preview'; raw TrueSkill rating 'elo' 1350.99 (sigma 7.68, 95% CI 1335.94-1366.05); normalised display value elo_norm not used. Pairwise LLM-judge comparisons of multi-turn role-play/analysis transcripts (README at the pinned commit names Claude Opus 4.6 as the default judge; possible self-preference for Anthropic models). Reasoning setting not stated in the data. Identity: catalogue hivemind-32b-preview (HF PrentisAI/hivemind-32b-preview). Re-read 2026-10-11 (elo 1350.99, sigma 7.68). Held at the 2026-10-10 merge (single EQ-Bench 3 result); imported under the 2026-10-11 owner decision.",
    "source_id": "eqbench3-canonical-results"
  },
  {
    "model": "bielik-11b-v2-0-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.6824,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2024-06-19",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'models/gwint2', model path speakleash/Bielik-11B-v2.0-Instruct; eq-bench_v2_pl score 68.24 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-06-19 11:21:15. Identity: catalogue bielik-11b-v2-0-instruct (HF speakleash/Bielik-11B-v2.0-Instruct); the only eq-bench_v2_pl run for this model path. SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab. The run finished before the public HF repo was created (2024-08-26, HF API createdAt), i.e. SpeakLeash ran its own pre-release copy and filed it under the released model path; held in pass 10, imported under the 2026-10-11 owner decision (result now in the Apache-2.0 Space).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v2-1-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.6627,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2024-08-24",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik_v2.1', model path speakleash/Bielik-11B-v2.1-Instruct; eq-bench_v2_pl score 66.27 (divided by 100); 155.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-08-24 10:07:46. Identity: catalogue bielik-11b-v2-1-instruct (HF speakleash/Bielik-11B-v2.1-Instruct); the only eq-bench_v2_pl run for this model path. SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab. The run finished before the public HF repo was created (2024-08-26, HF API createdAt), i.e. SpeakLeash ran its own pre-release copy and filed it under the released model path; held in pass 10, imported under the 2026-10-11 owner decision (result now in the Apache-2.0 Space).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v2-2-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.6905,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2024-08-24",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik_v2.2b', model path speakleash/Bielik-11B-v2.2-Instruct; eq-bench_v2_pl score 69.05 (divided by 100); 171.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2024-08-24 09:54:33. Identity: catalogue bielik-11b-v2-2-instruct (HF speakleash/Bielik-11B-v2.2-Instruct); the only eq-bench_v2_pl run for this model path. SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab. The run finished before the public HF repo was created (2024-08-26, HF API createdAt), i.e. SpeakLeash ran its own pre-release copy and filed it under the released model path; held in pass 10, imported under the 2026-10-11 owner decision (result now in the Apache-2.0 Space).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "bielik-11b-v2-6-instruct",
    "bench": "polish-eq-bench-v2",
    "score": 0.7413,
    "source": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv",
    "by": "lab",
    "date": "2025-05-30",
    "note": "SpeakLeash Polish EQ-Bench row run_id 'Bielik-11B-v2.6-Instruct', model path speakleash/Bielik-11B-v2.6-Instruct; eq-bench_v2_pl score 74.13 (divided by 100); 170.0 of 171 questions parseable; 1 iteration(s); inference engine transformers; completed 2025-05-30 15:56:57. Identity: catalogue bielik-11b-v2-6-instruct (HF speakleash/Bielik-11B-v2.6-Instruct); the only eq-bench_v2_pl run for this model path. SpeakLeash's own model on SpeakLeash's own benchmark Space, so by: lab. The run finished before the public HF repo was created (2025-06-02, HF API createdAt), i.e. SpeakLeash ran its own pre-release copy and filed it under the released model path; held in pass 10, imported under the 2026-10-11 owner decision (result now in the Apache-2.0 Space).",
    "source_id": "speakleash-polish-eq-bench"
  },
  {
    "model": "llama-7b",
    "bench": "arc-challenge",
    "score": 0.4462457337883959,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "arc-easy",
    "score": 0.7285353535353535,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "boolq",
    "score": 0.7504587155963303,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "hellaswag",
    "score": 0.7621987651862179,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "lambada-openai",
    "score": 0.7354938870560839,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "openbookqa",
    "score": 0.344,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "piqa",
    "score": 0.7867247007616975,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "sciq",
    "score": 0.946,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "social-iqa",
    "score": 0.4483111566018424,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "llama-7b",
    "bench": "winogrande",
    "score": 0.6992896606156275,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/huggyllama-llama-7b.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config use_fast=False,pretrained=huggyllama/llama-7b,trust_remote_code=True,low_cpu_mem_usage=True,dtype=auto. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: huggyllama/llama-7b is a community Hugging Face conversion of Meta's original LLaMA 7B weights (same weights in transformers format; Meta never published an HF repo for LLaMA 1), so mapped to catalogue llama-7b. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-challenge",
    "score": 0.5435153583617748,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_challenge.acc_norm; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "arc-easy",
    "score": 0.7954545454545454,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.arc_easy.acc_norm; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "boolq",
    "score": 0.8363914373088684,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.boolq.acc; task version 1; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "hellaswag",
    "score": 0.8105954989046007,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.hellaswag.acc_norm; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "lambada-openai",
    "score": 0.7568406753347564,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.lambada_openai.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "openbookqa",
    "score": 0.328,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.openbookqa.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "piqa",
    "score": 0.8079434167573449,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.piqa.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "sciq",
    "score": 0.959,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.sciq.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "social-iqa",
    "score": 0.4273285568065507,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.siqa.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "winogrande",
    "score": 0.7403314917127072,
    "source": "https://github.com/Stability-AI/StableLM/blob/93eea082c49c61e64da1a0e1092fe82cc74b46d3/evals/external/kittn_mistral-7B-v0.1-hf.json",
    "by": "independent",
    "date": "2024-04-08",
    "note": "Original full-dataset zero-shot lm-evaluation-harness run; JSON results.winogrande.acc; task version 0; model config pretrained=kittn/mistral-7B-v0.1-hf,dtype=bfloat16,trust_remote_code=True,low_cpu_mem_usage=True,use_fast=False. ARC/HellaSwag use byte-length-normalized accuracy; other tasks use accuracy. Stability AI's run of another lab's model (evals/external). Identity: kittn/mistral-7B-v0.1-hf (created 2023-09-27, the day Mistral AI released the weights) is a Hugging Face-format conversion of the Mistral 7B v0.1 weights (same weights, a third-party upload rather than mistralai/Mistral-7B-v0.1), so mapped to catalogue mistral-7b-v0-1. Held in pass 10 (not the maker's repository); imported under the 2026-10-11 owner decision. No run date published (date is repository snapshot).",
    "source_id": "stablelm-original-eval-json"
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "open-pt-llm-avg9",
    "score": 0.6994,
    "source": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz",
    "by": "independent",
    "date": "2024-03-08",
    "note": "Open PT-LLM Leaderboard row 'Gemini 1.0 Pro [2024-03-08]' (source=external, link https://ai.google.dev/, precision=?); published average 69.94 (divided by 100). API run (external) via the Gemini API, 2024-03-08. Identity: the Gemini API changelog lists only one Gemini 1.0 Pro model up to that date ('December 13 2023 ... gemini-pro: New text model'; the next 1.0 Pro entry is 'March 19, 2024 ... tuning Gemini 1.0 Pro'), i.e. gemini-1.0-pro-001, and no gemini-1.0-pro-002 release appears in the Gemini API changelog, so the unversioned run maps to catalogue gemini-1-0-pro-001. Held in pass 10 as an alias run; imported under the 2026-10-11 owner decision. Date is the leaderboard's row date (evaluation/submission), not model release.",
    "source_id": "open-pt-llm-leaderboard"
  },
  {
    "model": "minicpm3-4b",
    "bench": "livecodebench-official-20240101-20241101",
    "score": 0.215,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json",
    "by": "independent",
    "date": "2024-12-26",
    "note": "LiveCodeBench author-maintained website release; entry 'MiniCPM3-4B'; model_name=openbmb/MiniCPM3-4B. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=21.5245478036%; website rounding=21.5%; divided by 100. Exact upstream model_name matched to original catalogue (catalogue minicpm3-4b = openbmb/MiniCPM3-4B). Same window and the same 387 problem IDs as the existing rows; method reproduces deepseek-v3's 48.9405684755%. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Added in pass 12 (held in pass 10 only because the extract files were missing).",
    "source_id": "livecodebench-website-20241226"
  },
  {
    "model": "gemini-2-5-pro-exp-03-25",
    "bench": "livecodebench-official-20231101-20250201",
    "score": 0.809,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Gemini-2.5-Pro'; model_name=gemini-2.5-pro-exp-03-25. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=80.9220985692%; website rounding=80.9%; divided by 100. Identity: upstream model_name 'gemini-2.5-pro-exp-03-25' equals the catalogued dated experimental id. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "qwq-32b-preview",
    "bench": "livecodebench-official-20231101-20250201",
    "score": 0.558,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'QwQ-32B-Preview'; model_name=Qwen/QwQ-32B-Preview. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=55.8028616852%; website rounding=55.8%; divided by 100. Identity: upstream model_name 'Qwen/QwQ-32B-Preview' equals the catalogued repo. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "livecodebench-official-20231101-20250201",
    "score": 0.379,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Gemini-Flash-2.0-Exp'; model_name=gemini-2.0-flash-exp. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=37.8537360890%; website rounding=37.9%; divided by 100. Identity: upstream model_name 'gemini-2.0-flash-exp' equals the catalogued experimental id. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "llama-3-1-nemotron-ultra-253b",
    "bench": "livecodebench-official-20231101-20250201",
    "score": 0.749,
    "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json",
    "by": "independent",
    "date": "2025-08-01",
    "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Llama-3_1-Nemotron-Ultra-253B-v1'; model_name=nvidia/Llama-3_1-Nemotron-Ultra-253B-v1. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=74.8807631161%; website rounding=74.9%; divided by 100. Identity: upstream model_name 'nvidia/Llama-3_1-Nemotron-Ultra-253B-v1' equals the catalogued repo; no vendor submission PR for it exists in LiveCodeBench/submissions (the frozen 2026-09 review withheld it because its own-run origin was not established; imported under the 2026-10-11 owner decision). One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.",
    "source_id": "livecodebench-website-20250801"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1297,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4o-2024-05-13' (rank 1), column Rating = 1297 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-3-5-sonnet-20240620",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1286,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-3-5-sonnet-20240620' (rank 2), column Rating = 1286 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1273,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-3-opus-20240229' (rank 5), column Rating = 1273 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-1106-preview",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1265,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4-1106-preview' (rank 7), column Rating = 1265 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1258,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4-turbo-2024-04-09' (rank 9), column Rating = 1258 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-0125-preview",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1255,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4-0125-preview' (rank 10), column Rating = 1255 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-2-27b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1232,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-2-27b-it' (rank 13), column Rating = 1232 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "nemotron-4-340b-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1222,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'nemotron-4-340b-instruct' (rank 15), column Rating = 1222 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1220,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-3-sonnet-20240229' (rank 16), column Rating = 1220 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-0314",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1213,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4-0314' (rank 18), column Rating = 1213 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1209,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-3-haiku-20240307' (rank 20), column Rating = 1209 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-2-9b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1204,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-2-9b-it' (rank 21), column Rating = 1204 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "glm-4-0520",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1202,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'glm-4-0520' (rank 22), column Rating = 1202 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-0613",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1191,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-4-0613' (rank 23), column Rating = 1191 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-3-70b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1187,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-3-70b-instruct' (rank 26), column Rating = 1187 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen2-72b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1182,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen2-72b-instruct' (rank 28), column Rating = 1182 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "deepseek-coder-v2",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1181,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'deepseek-coder-v2' (rank 30), column Rating = 1181 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1170,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'reka-flash-21b-20240226' (rank 33), column Rating = 1170 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-2-0",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1164,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-2.0' (rank 34), column Rating = 1164 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mistral-large-2402",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1163,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mistral-large-2402' (rank 35), column Rating = 1163 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-110b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1161,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-110b-chat' (rank 37), column Rating = 1161 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-3-5-turbo-0613",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1161,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-3.5-turbo-0613' (rank 38), column Rating = 1161 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "claude-2-1",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1156,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'claude-2.1' (rank 39), column Rating = 1156 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mixtral-8x22b-instruct-v0-1",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1152,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mixtral-8x22b-instruct-v0.1' (rank 42), column Rating = 1152 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-3-8b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1150,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-3-8b-instruct' (rank 43), column Rating = 1150 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-72b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1148,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-72b-chat' (rank 45), column Rating = 1148 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-3-5-turbo-0125",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1147,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-3.5-turbo-0125' (rank 46), column Rating = 1147 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "zephyr-orpo-141b-a35b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1143,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'zephyr-orpo-141b-A35b-v0.1' (rank 47), column Rating = 1143 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "snowflake-arctic-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1126,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'snowflake-arctic-instruct' (rank 51), column Rating = 1126 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-32b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1126,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-32b-chat' (rank 52), column Rating = 1126 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "yi-1-5-34b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1126,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'yi-1.5-34b-chat' (rank 53), column Rating = 1126 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "phi-3-medium-4k-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1126,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'phi-3-medium-4k-instruct' (rank 54), column Rating = 1126 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "tulu-2-dpo-70b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1122,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'tulu-2-dpo-70b' (rank 55), column Rating = 1122 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mixtral-8x7b-instruct-v0-1",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1114,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mixtral-8x7b-instruct-v0.1' (rank 56), column Rating = 1114 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "openchat-3-5-0106",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1114,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'openchat-3.5-0106' (rank 57), column Rating = 1114 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-14b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1112,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-14b-chat' (rank 58), column Rating = 1112 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama2-70b-steerlm-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1111,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama2-70b-steerlm-chat' (rank 59), column Rating = 1111 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1111,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'starling-lm-7b-beta' (rank 60), column Rating = 1111 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-2-70b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1108,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-2-70b-chat' (rank 61), column Rating = 1108 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-3-5-turbo-1106",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1106,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt-3.5-turbo-1106' (rank 62), column Rating = 1106 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "phi-3-small-8k-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1103,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'phi-3-small-8k-instruct' (rank 64), column Rating = 1103 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "openchat-3-5",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1101,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'openchat-3.5' (rank 65), column Rating = 1101 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "yi-34b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1101,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'yi-34b-chat' (rank 67), column Rating = 1101 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1099,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'starling-lm-7b-alpha' (rank 68), column Rating = 1099 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1097,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-1.1-7b-it' (rank 69), column Rating = 1097 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "deepseek-llm-67b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1092,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'deepseek-llm-67b-chat' (rank 71), column Rating = 1092 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "nous-hermes-2-mixtral-8x7b-dpo",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1090,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'nous-hermes-2-mixtral-8x7b-dpo' (rank 72), column Rating = 1090 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1086,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-7b-chat' (rank 73), column Rating = 1086 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1081,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-2-13b-chat' (rank 75), column Rating = 1081 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen-14b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1081,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen-14b-chat' (rank 76), column Rating = 1081 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1075,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'openhermes-2.5-mistral-7b' (rank 78), column Rating = 1075 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "phi-3-mini-128k-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1072,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'phi-3-mini-128k-instruct' (rank 79), column Rating = 1072 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "codellama-34b-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1070,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'codellama-34b-instruct' (rank 80), column Rating = 1070 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "phi-3-mini-4k-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1068,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'phi-3-mini-4k-instruct' (rank 81), column Rating = 1068 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1064,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'solar-10.7b-instruct-v1.0' (rank 82), column Rating = 1064 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "dolphin-2-2-1-mistral-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1060,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'dolphin-2.2.1-mistral-7b' (rank 83), column Rating = 1060 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "falcon-180b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1056,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'falcon-180b-chat' (rank 85), column Rating = 1056 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1054,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mistral-7b-instruct-v0.2' (rank 86), column Rating = 1054 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1051,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'zephyr-7b-alpha' (rank 87), column Rating = 1051 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1049,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'zephyr-7b-beta' (rank 88), column Rating = 1049 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-1-1-2b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1044,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-1.1-2b-it' (rank 89), column Rating = 1044 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mpt-30b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1041,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mpt-30b-chat' (rank 90), column Rating = 1041 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "codellama-70b-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1039,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'codellama-70b-instruct' (rank 91), column Rating = 1039 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1036,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-2-7b-chat' (rank 93), column Rating = 1036 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "guanaco-33b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1035,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'guanaco-33b' (rank 94), column Rating = 1035 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1029,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-7b-it' (rank 95), column Rating = 1029 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "stripedhyena-nous-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1028,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'stripedhyena-nous-7b' (rank 96), column Rating = 1028 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "qwen1-5-4b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1026,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'qwen1.5-4b-chat' (rank 97), column Rating = 1026 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemma-2b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 995,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemma-2b-it' (rank 100), column Rating = 995 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "olmo-7b-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 995,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'olmo-7b-instruct' (rank 101), column Rating = 995 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "rwkv-4-raven-14b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 971,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'RWKV-4-Raven-14B' (rank 102), column Rating = 971 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "koala-13b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 967,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'koala-13b' (rank 103), column Rating = 967 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "alpaca-13b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 955,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'alpaca-13b' (rank 104), column Rating = 955 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "chatglm3-6b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 946,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'chatglm3-6b' (rank 105), column Rating = 946 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mpt-7b-chat",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 944,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mpt-7b-chat' (rank 106), column Rating = 944 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "chatglm2-6b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 930,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'chatglm2-6b' (rank 107), column Rating = 930 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt4all-13b-snoozy",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 924,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gpt4all-13b-snoozy' (rank 108), column Rating = 924 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "fastchat-t5-3b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 879,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'fastchat-t5-3b' (rank 110), column Rating = 879 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "chatglm-6b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 874,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'chatglm-6b' (rank 111), column Rating = 874 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "dolly-v2-12b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 856,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'dolly-v2-12b' (rank 112), column Rating = 856 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "llama-13b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 853,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'llama-13b' (rank 113), column Rating = 853 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "stablelm-tuned-alpha-7b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 843,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'stablelm-tuned-alpha-7b' (rank 114), column Rating = 843 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label is the Arena model key, matched to the catalogue model by exact id/name.",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "command-r-plus-04-2024",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1214,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'command-r-plus' (rank 17), column Rating = 1214 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'command-r-plus' mapped to command-r-plus-04-2024 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'command-r-plus\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gemini-1-0-pro-001",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1182,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'gemini-pro-dev-api' (rank 29), column Rating = 1182 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'gemini-pro-dev-api' mapped to gemini-1-0-pro-001 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'gemini-pro-dev-api\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "command-r-03-2024",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1175,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'command-r' (rank 32), column Rating = 1175 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'command-r' mapped to command-r-03-2024 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'command-r\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mistral-medium-2312",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1153,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mistral-medium' (rank 41), column Rating = 1153 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'mistral-medium' mapped to mistral-medium-2312 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'mistral-medium\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "wizardlm-70b-v1-0",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1129,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'wizardlm-70b' (rank 50), column Rating = 1129 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'wizardlm-70b' mapped to wizardlm-70b-v1-0 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'wizardlm-70b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "vicuna-33b-v1-3",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1105,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'vicuna-33b' (rank 63), column Rating = 1105 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'vicuna-33b' mapped to vicuna-33b-v1-3 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'vicuna-33b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "dbrx-instruct",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1101,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'dbrx-instruct-preview' (rank 66), column Rating = 1101 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'dbrx-instruct-preview' mapped to dbrx-instruct exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'dbrx-instruct-preview\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "wizardlm-13b-v1-2",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1083,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'wizardlm-13b' (rank 74), column Rating = 1083 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'wizardlm-13b' mapped to wizardlm-13b-v1-2 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'wizardlm-13b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "vicuna-13b-v1-5",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1077,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'vicuna-13b' (rank 77), column Rating = 1077 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'vicuna-13b' mapped to vicuna-13b-v1-5 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'vicuna-13b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1058,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'vicuna-7b' (rank 84), column Rating = 1058 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'vicuna-7b' mapped to vicuna-7b-v1-5 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'vicuna-7b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1008,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'mistral-7b-instruct' (rank 98), column Rating = 1008 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'mistral-7b-instruct' mapped to mistral-7b-instruct-v0-1 exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'mistral-7b-instruct\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "palm-2-chat-bison",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 997,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'palm-2' (rank 99), column Rating = 997 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'palm-2' mapped to palm-2-chat-bison exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'palm-2\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "oasst-sft-4-pythia-12b",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 912,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'oasst-pythia-12b' (rank 109), column Rating = 912 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Arena key 'oasst-pythia-12b' mapped to oasst-sft-4-pythia-12b exactly as our lmarena-text rows map the same Arena key (lmarena-dataset note 'exact entry \\'oasst-pythia-12b\\'').",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "reka-core-20240501",
    "bench": "polyrating-chatbot-arena-rating",
    "score": 1212,
    "source": "https://arxiv.org/html/2409.00696v3#A6.T7",
    "by": "independent",
    "date": "2025-02-11",
    "note": "Polyrating paper (arXiv 2409.00696v3, CC BY 4.0) Table 7 'Leaderboard of human evaluation with modifiers fitted with Polyrating', row 'reka-core-20240501' (rank 19), column Rating = 1212 (re-read 2026-10-11 from the v3 HTML). The authors' own multidimensional Bradley-Terry fit of the public Chatbot Arena human votes (Chiang et al. 2024), i.e. the same underlying votes as lmarena-text but a different rating model. Row label equals the catalogue id reka-core-20240501 (added to the catalogue after pass 4).",
    "source_id": "paper-2409.00696v3-polyrating"
  },
  {
    "model": "gpt-4-1",
    "bench": "charxiv-rq",
    "score": 0.567,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4.1', printed '56.7%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-mini",
    "bench": "charxiv-rq",
    "score": 0.568,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4.1 mini', printed '56.8%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-1-nano-2025-04-14",
    "bench": "charxiv-rq",
    "score": 0.405,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4.1 nano', printed '40.5%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4o-2024-11-20",
    "bench": "charxiv-rq",
    "score": 0.527,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4o (2024-11-20)', printed '52.7%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4o-mini-2024-07-18",
    "bench": "charxiv-rq",
    "score": 0.368,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4o mini', printed '36.8%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "o1-2024-12-17",
    "bench": "charxiv-rq",
    "score": 0.551,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'OpenAI o1 (high)', printed '55.1%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Reasoning effort high (column label). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-4-5-preview-2025-02-27",
    "bench": "charxiv-rq",
    "score": 0.554,
    "source": "https://openai.com/index/gpt-4-1",
    "by": "lab",
    "date": "2025-04-14",
    "note": "OpenAI 'Introducing GPT-4.1 in the API' (April 14, 2025), appendix table 'Vision' (columns GPT-4.1 | GPT-4.1 mini | GPT-4.1 nano | GPT-4o (2024-11-20) | GPT-4o mini | OpenAI o1 (high) | OpenAI o3-mini (high) | GPT-4.5), row 'CharXiv-R', column 'GPT-4.5', printed '55.4%'; re-read 2026-10-11 from Internet Archive capture 20261006095914 (openai.com returns HTTP 403). Tool setting: the GPT-4.1 post states none, but OpenAI's 'Introducing GPT-5 for developers' (Aug 7, 2025; capture 20261006230110) reprints the identical GPT-4.1 / 4.1 mini / 4.1 nano figures (56.7 / 56.8 / 40.5) in a row labelled 'CharXiv reasoning (python enabled)', so this table's CharXiv-R cells are most likely python-enabled runs (same harness for every column). Recorded under charxiv-rq (the only CharXiv-reasoning id; vendor rows may only use existing benchmark ids) with the setting stated here. Split not stated (CharXiv default: validation). Vendor-reported (rule b): OpenAI's own figure for its own model, hand-recorded. The CharXiv-D cells of the same table are already in data.",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-5-instant",
    "bench": "charxiv-rq",
    "score": 0.816,
    "source": "https://openai.com/index/gpt-5-5-instant",
    "by": "lab",
    "date": "2026-05-05",
    "note": "OpenAI 'GPT-5.5 Instant' page, bar chart 'CharXiv-reasoning — Scientific Chart Reasoning' (subtitle 'Instant models'), bar 'GPT-5.5 Instant' with printed data label '81.6%' (the page's Vega-Lite chart renders score_label = format(score, '.1f') + '%' as text on each bar; data value 81.6). Read from Internet Archive capture 20260923222921 (openai.com returns HTTP 403). Instant (non-reasoning) model; tool use not stated. The other bar (GPT-5.3 Instant 75.0) is not used: no catalogue entry with certain identity. Vendor-reported (rule b); printed chart data label (owner policy 2026-10-09).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "exaone-4-5-33b",
    "bench": "charxiv-rq",
    "score": 0.717,
    "source": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.5-33B/blob/570aa4b15a4f45ba1133072b45f50198f6e3b4fd/README.md#evaluation-results",
    "by": "lab",
    "date": "2026-04-04",
    "note": "LG AI Research EXAONE-4.5-33B model card (pinned 570aa4b = current Hub sha on 2026-10-09), Evaluation Results > Vision-Language Tasks HTML table, row 'CharXiv (RQ)', column 'EXAONE 4.5 33B (Reasoning)' printed 71.7 (text cell, not an image). Reasoning mode (the card's highest setting; enable_thinking=True is the default). No tool use mentioned; split not stated (CharXiv authors' default is validation). Checkpoint pinned by the card itself (repo LGAI-EXAONE/EXAONE-4.5-33B = catalogue exaone-4-5-33b). Date = model repository creation date, as in the existing LG rows. Vendor-reported (rule b): EXAONE model licence is not an open licence; only LG's own column used (GPT-5 mini, Qwen3-VL 32B/235B Thinking and Qwen3.5 27B columns not used). Single-model run for first-party-lgai-scores x charxiv-rq (recorded anyway, owner decision 2026-10-11). Re-read 2026-10-11 at the pinned raw README (row 'CharXiv (RQ)', first column 'EXAONE 4.5 33B (Reasoning)' = 71.7).",
    "source_id": "first-party-lgai-scores"
  },
  {
    "model": "ernie-5-0-0110",
    "bench": "charxiv-rq",
    "score": 0.671,
    "source": "https://arxiv.org/html/2602.04705v1#S6.T3",
    "by": "lab",
    "date": "2026-02-04",
    "note": "Baidu ERNIE Team, ERNIE 5.0 Technical Report, arXiv:2602.04705v1 (CC BY 4.0), Sec. 6.2 Table 3 'Evaluation of post-trained models across a wide range of vision benchmarks', row 'ChartXiv-RQ' (the report's spelling of CharXiv), column 'ERNIE 5.0' = 67.10; re-read 2026-10-11. Evaluated with Baidu's internal framework ERNIE-Eval (Sec. 6). Split not stated. Identity: the report prints no snapshot; ernie-5-0-0110 is the ERNIE 5.0 release current when the report was submitted (2026-02-04); the 2025 'Preview' snapshots are catalogued separately.",
    "source_id": "report-2602.04705v1"
  },
  {
    "model": "ernie-5-0-0110",
    "bench": "charxiv-dq",
    "score": 0.8905,
    "source": "https://arxiv.org/html/2602.04705v1#S6.T3",
    "by": "lab",
    "date": "2026-02-04",
    "note": "Baidu ERNIE Team, ERNIE 5.0 Technical Report, arXiv:2602.04705v1 (CC BY 4.0), Sec. 6.2 Table 3 'Evaluation of post-trained models across a wide range of vision benchmarks', row 'ChartXiv-DQ' (the report's spelling of CharXiv), column 'ERNIE 5.0' = 89.05; re-read 2026-10-11. Evaluated with Baidu's internal framework ERNIE-Eval (Sec. 6). Split not stated. Identity: the report prints no snapshot; ernie-5-0-0110 is the ERNIE 5.0 release current when the report was submitted (2026-02-04); the 2025 'Preview' snapshots are catalogued separately.",
    "source_id": "report-2602.04705v1"
  },
  {
    "model": "internvl2-4b",
    "bench": "charxiv-rq",
    "score": 0.245,
    "source": "https://arxiv.org/html/2412.05271v5#S5.T7",
    "by": "lab",
    "date": "2025-09-26",
    "note": "OpenGVLab / Shanghai AI Laboratory, InternVL2.5 report (arXiv 2412.05271v5), Table 7 'Comparison of OCR, chart, and document understanding performance', row 'InternVL2-4B [35]', column 'CharXiv (RQ / DQ)' = 24.5 / 48.0, this row's RQ value 24.5; re-read 2026-10-11. Caption: 'Part of results are collected from [64, 54, 8, 257, 302] and the OpenCompass leaderboard [46]' (OpenCompass is Shanghai AI Lab's own); the CharXiv authors' README [257] has no InternVL2-4B row, so this is OpenGVLab's own figure. arXiv non-exclusive licence: vendor-reported own-model cell only (rule b). Checkpoint OpenGVLab/InternVL2-4B (catalogue internvl2-4b). Single-model run under this ledger entry.",
    "source_id": "first-party-opengvlab-scores"
  },
  {
    "model": "internvl2-4b",
    "bench": "charxiv-dq",
    "score": 0.48,
    "source": "https://arxiv.org/html/2412.05271v5#S5.T7",
    "by": "lab",
    "date": "2025-09-26",
    "note": "OpenGVLab / Shanghai AI Laboratory, InternVL2.5 report (arXiv 2412.05271v5), Table 7 'Comparison of OCR, chart, and document understanding performance', row 'InternVL2-4B [35]', column 'CharXiv (RQ / DQ)' = 24.5 / 48.0, this row's DQ value 48.0; re-read 2026-10-11. Caption: 'Part of results are collected from [64, 54, 8, 257, 302] and the OpenCompass leaderboard [46]' (OpenCompass is Shanghai AI Lab's own); the CharXiv authors' README [257] has no InternVL2-4B row, so this is OpenGVLab's own figure. arXiv non-exclusive licence: vendor-reported own-model cell only (rule b). Checkpoint OpenGVLab/InternVL2-4B (catalogue internvl2-4b). Single-model run under this ledger entry.",
    "source_id": "first-party-opengvlab-scores"
  },
  {
    "model": "gpt-5",
    "bench": "charxiv-dq",
    "score": 0.9117,
    "source": "https://arxiv.org/html/2602.04705v1#S6.T3",
    "by": "independent",
    "date": "2026-02-04",
    "note": "Baidu ERNIE Team, ERNIE 5.0 Technical Report, arXiv:2602.04705v1 (CC BY 4.0), Sec. 6.2 Table 3, row 'ChartXiv-DQ', column 'GPT-5 (High)' = 91.17; re-read 2026-10-11. Provenance: Sec. 6 'We conduct systematic evaluations of ERNIE 5.0 against state-of-the-art models ... with internal evaluation framework, ERNIE-Eval'; no maker publishes CharXiv-DQ for this model and the CharXiv authors' leaderboard predates it, so the cell is read as Baidu's own run (the same table's RQ cells for GPT-5 and Gemini 3-Pro equal the makers' figures and are not used). Identity: GPT-5 (High) = API gpt-5 at reasoning effort high (catalogue gpt-5). Split not stated.",
    "source_id": "report-2602.04705v1"
  },
  {
    "model": "gemini-2-5-pro",
    "bench": "charxiv-dq",
    "score": 0.9338,
    "source": "https://arxiv.org/html/2602.04705v1#S6.T3",
    "by": "independent",
    "date": "2026-02-04",
    "note": "Baidu ERNIE Team, ERNIE 5.0 Technical Report, arXiv:2602.04705v1 (CC BY 4.0), Sec. 6.2 Table 3, row 'ChartXiv-DQ', column 'Gemini 2.5-Pro' = 93.38; re-read 2026-10-11. Provenance: Sec. 6 'We conduct systematic evaluations of ERNIE 5.0 against state-of-the-art models ... with internal evaluation framework, ERNIE-Eval'; no maker publishes CharXiv-DQ for this model and the CharXiv authors' leaderboard predates it, so the cell is read as Baidu's own run (the same table's RQ cells for GPT-5 and Gemini 3-Pro equal the makers' figures and are not used). Identity: snapshot not printed; the GA Gemini 2.5 Pro was the only 2.5 Pro served when the report was written (Feb 2026), so catalogue gemini-2-5-pro. Split not stated.",
    "source_id": "report-2602.04705v1"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "charxiv-dq",
    "score": 0.9595,
    "source": "https://arxiv.org/html/2602.04705v1#S6.T3",
    "by": "independent",
    "date": "2026-02-04",
    "note": "Baidu ERNIE Team, ERNIE 5.0 Technical Report, arXiv:2602.04705v1 (CC BY 4.0), Sec. 6.2 Table 3, row 'ChartXiv-DQ', column 'Gemini 3-Pro' = 95.95; re-read 2026-10-11. Provenance: Sec. 6 'We conduct systematic evaluations of ERNIE 5.0 against state-of-the-art models ... with internal evaluation framework, ERNIE-Eval'; no maker publishes CharXiv-DQ for this model and the CharXiv authors' leaderboard predates it, so the cell is read as Baidu's own run (the same table's RQ cells for GPT-5 and Gemini 3-Pro equal the makers' figures and are not used). Identity: the only Gemini 3 Pro available before Feb 2026 was gemini-3-pro-preview (catalogue gemini-3-pro-preview). Split not stated.",
    "source_id": "report-2602.04705v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "charxiv-rq-test",
    "score": 0.4701,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'GPT-4o [1]', column 'Reasoning Questions / All' = 47.01; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': gpt-4o-2024-05-13. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "charxiv-dq-test",
    "score": 0.8492,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'GPT-4o [1]', column 'Descriptive Questions / All' = 84.92; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': gpt-4o-2024-05-13. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "charxiv-rq-test",
    "score": 0.3379,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'GPT-4V [1]', column 'Reasoning Questions / All' = 33.79; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': gpt-4-turbo-2024-04-09. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "charxiv-dq-test",
    "score": 0.7978,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'GPT-4V [1]', column 'Descriptive Questions / All' = 79.78; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': gpt-4-turbo-2024-04-09. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "charxiv-rq-test",
    "score": 0.3235,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Sonnet [3]', column 'Reasoning Questions / All' = 32.35; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-sonnet-20240229. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-sonnet-20240229",
    "bench": "charxiv-dq-test",
    "score": 0.7275,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Sonnet [3]', column 'Descriptive Questions / All' = 72.75; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-sonnet-20240229. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "charxiv-rq-test",
    "score": 0.3046,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Haiku [3]', column 'Reasoning Questions / All' = 30.46; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-haiku-20240307. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-haiku-20240307",
    "bench": "charxiv-dq-test",
    "score": 0.6449,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Haiku [3]', column 'Descriptive Questions / All' = 64.49; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-haiku-20240307. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "charxiv-rq-test",
    "score": 0.288,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Opus [3]', column 'Reasoning Questions / All' = 28.80; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-opus-20240229. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "claude-3-opus-20240229",
    "bench": "charxiv-dq-test",
    "score": 0.7222,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Claude 3 Opus [3]', column 'Descriptive Questions / All' = 72.22; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': claude-3-opus-20240229. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "charxiv-rq-test",
    "score": 0.2714,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Reka Flash [52]', column 'Reasoning Questions / All' = 27.14; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': reka-flash-20240226. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "reka-flash-21b-20240226",
    "bench": "charxiv-dq-test",
    "score": 0.5472,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'Reka Flash [52]', column 'Descriptive Questions / All' = 54.72; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': reka-flash-20240226. Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "charxiv-rq-test",
    "score": 0.2003,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'LLaVA 1.6 Yi 34B [41]', column 'Reasoning Questions / All' = 20.03; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': llava-hf/llava-v1.6-34b-hf (HF conversion of liuhaotian/llava-v1.6-34b, as for the README rows). Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "llava-v1-6-34b",
    "bench": "charxiv-dq-test",
    "score": 0.5146,
    "source": "https://arxiv.org/html/2406.18521v1#A3.T4",
    "by": "independent",
    "date": "2024-06-26",
    "note": "CharXiv paper (Wang et al., arXiv:2406.18521v1, CC BY-SA 4.0), Appendix C Table 4 'Model evaluation results on test set', row 'LLaVA 1.6 Yi 34B [41]', column 'Descriptive Questions / All' = 51.46; re-read 2026-10-11. Test split: 1,323 charts with private answers, scored by the authors (their own runs of every model; the validation-set results of the same models are in charxiv-rq/charxiv-dq from the authors' README). Checkpoint from Table 11 'Run configurations for all models': llava-hf/llava-v1.6-34b-hf (HF conversion of liuhaotian/llava-v1.6-34b, as for the README rows). Share-alike: also listed in this file's extracts.",
    "source_id": "paper-2406.18521v1-charxiv"
  },
  {
    "model": "minimax-m2-1",
    "bench": "tau2-telecom",
    "score": 0.87,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M2.1/blob/cd97f59135f37b2a6bf09356e485d5e4aeb7dc9c/README.md",
    "by": "lab",
    "date": "2026-02-13",
    "note": "MiniMax-M2.1 model card @cd97f59 (README revision of 2026-02-13), 'Benchmarks' last table, row '𝜏²-Bench Telecom', column 'MiniMax-M2.1' = 87.0; re-read 2026-10-11. Card note: 'AIME25 ~ 𝜏²-Bench Telecom: Derived from internal testing based on the evaluation datasets and methodology referenced in the Artificial Analysis Intelligence Index' (MiniMax's own run following AA's published methodology; not an AA-sourced number). Vendor-reported (rule b), own models only. Comparator columns (Claude Sonnet 4.5 78.0, Claude Opus 4.5 90.0, Gemini 3 Pro 87.0, GPT-5.2 85.0, DeepSeek V3.2 91.0) not used: Sonnet 4.5's 78 repeats the figure the MiniMax-M2 card credits to artificialanalysis.ai, so the column cells may be AA copies.",
    "source_id": "first-party-minimax-scores"
  },
  {
    "model": "minimax-m2",
    "bench": "tau2-telecom",
    "score": 0.87,
    "source": "https://huggingface.co/MiniMaxAI/MiniMax-M2.1/blob/cd97f59135f37b2a6bf09356e485d5e4aeb7dc9c/README.md",
    "by": "lab",
    "date": "2026-02-13",
    "note": "MiniMax-M2.1 model card @cd97f59 (README revision of 2026-02-13), 'Benchmarks' last table, row '𝜏²-Bench Telecom', column 'MiniMax-M2' = 87.0; re-read 2026-10-11. Card note: 'AIME25 ~ 𝜏²-Bench Telecom: Derived from internal testing based on the evaluation datasets and methodology referenced in the Artificial Analysis Intelligence Index' (MiniMax's own run following AA's published methodology; not an AA-sourced number). Same-lab sibling column (MiniMax's previous model) in MiniMax's own table. Vendor-reported (rule b), own models only. Comparator columns (Claude Sonnet 4.5 78.0, Claude Opus 4.5 90.0, Gemini 3 Pro 87.0, GPT-5.2 85.0, DeepSeek V3.2 91.0) not used: Sonnet 4.5's 78 repeats the figure the MiniMax-M2 card credits to artificialanalysis.ai, so the column cells may be AA copies.",
    "source_id": "first-party-minimax-scores"
  },
  {
    "model": "nemotron-3-ultra-550b-a55b",
    "bench": "tau3-telecom",
    "score": 0.929,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "lab",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'N-3-Ultra 550B-A55B' = 92.9; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: own model (BF16), as for the existing rows of this table. The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "minimax-m2-7",
    "bench": "tau3-telecom",
    "score": 0.896,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'MiniMax-2.7 230B-A10B' = 89.6; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: catalogue minimax-m2-7. The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "glm-5-1",
    "bench": "tau3-telecom",
    "score": 0.969,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'GLM-5.1 744B-A40B' = 96.9; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: catalogue glm-5-1. The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "kimi-k2-6",
    "bench": "tau3-telecom",
    "score": 0.978,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'Kimi-K2.6 1T-A32B' = 97.8; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: catalogue kimi-k2-6. The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-5-397b-a17b",
    "bench": "tau3-telecom",
    "score": 0.98,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'Qwen-3.5 397B-17B' = 98.0; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: catalogue qwen3-5-397b-a17b. The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-pro-preview",
    "bench": "tau3-telecom",
    "score": 0.963,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'DS-v4-Pro 1.6T-A49B' = 96.3; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: the April 2026 DeepSeek-V4-Pro (the only V4 Pro released before this June 2026 report; same mapping as the existing rows of this table). The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "tau3-telecom",
    "score": 0.983,
    "source": "https://arxiv.org/html/2606.15007v1",
    "by": "independent",
    "date": "2026-06-12",
    "note": "NVIDIA Nemotron 3 Ultra report (arXiv 2606.15007v1, CC BY 4.0), Table 10 'Evaluation suite for Nemotron 3 Ultra. We compare against six open models.', row 'TauBench V3 / Telecom', column 'DS-v4-Flash 284B-A13B' = 98.3; re-read 2026-10-11. NVIDIA's own runs: 'All models were evaluated under the same evaluation settings including agentic resources ..., input data, prompt templates, number of repeats, and extraction/metric implementation'; Tau Bench 3 via official implementation or internal scaffolding; appendix: AGI-Eval 'tau2-bench-revised' user prompt added, user simulator GPT-5.2 (low reasoning effort), 8-trial average, DeepSeek-V4 models at max reasoning. Identity: the April 2026 DeepSeek-V4-Flash (same mapping as the existing rows of this table). The TauBench V3 Average row is already in data as tau3-bench; this domain row was left out earlier only to avoid correlated rows.",
    "source_id": "report-2606.15007v1"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-pro",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Knowledge', row 'MMLU-Pro', column 'Qwen3-32B Non-Thinking' = 71.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-redux",
    "score": 0.857,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-32B Non-Thinking' = 85.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "gpqa-unspecified",
    "score": 0.546,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-32B Non-Thinking' = 54.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "supergpqa",
    "score": 0.432,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-32B Non-Thinking' = 43.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "aime-2025",
    "score": 0.202,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Reasoning', row 'AIME25', column 'Qwen3-32B Non-Thinking' = 20.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.098,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-32B Non-Thinking' = 9.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "livebench-20241125",
    "score": 0.598,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-32B Non-Thinking' = 59.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "livecodebench-v6",
    "score": 0.291,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Coding', row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-32B Non-Thinking' = 29.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "multipl-e",
    "score": 0.769,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-32B Non-Thinking' = 76.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "aider-polyglot",
    "score": 0.4,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-32B Non-Thinking' = 40.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "ifeval-unspecified",
    "score": 0.832,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-32B Non-Thinking' = 83.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "arena-hard-v2",
    "score": 0.341,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-32B Non-Thinking' = 34.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "creative-writing-v3",
    "score": 0.783,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-32B Non-Thinking' = 78.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "writingbench",
    "score": 0.754,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-32B Non-Thinking' = 75.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "bfcl-v3",
    "score": 0.63,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-32B Non-Thinking' = 63.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-retail",
    "score": 0.401,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'TAU1-Retail', column 'Qwen3-32B Non-Thinking' = 40.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau-bench-airline",
    "score": 0.17,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'TAU1-Airline', column 'Qwen3-32B Non-Thinking' = 17.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-retail",
    "score": 0.488,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'TAU2-Retail', column 'Qwen3-32B Non-Thinking' = 48.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-airline",
    "score": 0.24,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'TAU2-Airline', column 'Qwen3-32B Non-Thinking' = 24.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "tau2-telecom",
    "score": 0.246,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Agent', row 'TAU2-Telecom', column 'Qwen3-32B Non-Thinking' = 24.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "multi-if-unspecified",
    "score": 0.707,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-32B Non-Thinking' = 70.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "mmlu-prox",
    "score": 0.693,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-32B Non-Thinking' = 69.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "include",
    "score": 0.709,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-32B Non-Thinking' = 70.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "polymath",
    "score": 0.225,
    "source": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen's own evaluation, pinned Qwen3-Next-80B-A3B-Instruct (@9c7f2fb) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-32B Non-Thinking' = 22.5; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column 'Qwen3-32B Non-Thinking' in Qwen's own table (catalogue qwen3-32b = the April 2025 hybrid checkpoint, thinking disabled). Earlier imports skipped this column under the thinking-only convention.",
    "source_id": "card-qwen--qwen3-next-80b-a3b-instruct"
  },
  {
    "model": "gpt-5-1",
    "bench": "tau2-telecom",
    "score": 0.956,
    "source": "https://openai.com/index/gpt-5-1-for-developers",
    "by": "lab",
    "date": "2025-11-13",
    "note": "OpenAI 'Introducing GPT-5.1 for developers' (November 13, 2025), 'Appendix: Model evaluations' table, row 'Tau2-bench Telecom*', column 'GPT-5.1 (high)', printed '95.6%'; re-read 2026-10-11 from Internet Archive capture 20260910181654 (openai.com returns HTTP 403). Reasoning effort high. SETTING DIFFERS: footnote '* For Tau2-bench Telecom, we gave GPT-5.1 a short, generically helpful prompt to improve its performance.' (not OpenAI's later 'original prompts' setup used for the GPT-5.4/5.5 rows of this run). User model not stated. Vendor-reported (rule b).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-2",
    "bench": "tau2-telecom",
    "score": 0.987,
    "source": "https://openai.com/index/introducing-gpt-5-2",
    "by": "lab",
    "date": "2025-12-11",
    "note": "OpenAI 'Introducing GPT-5.2' (December 11, 2025), Appendix 'Detailed benchmarks', 'Tool usage' table, row 'Tau2-bench Telecom', column 'GPT-5.2 Thinking', printed '98.7%' (body: 'GPT-5.2 Thinking achieves a new state of the art of 98.7% on Tau2-bench Telecom'); re-read 2026-10-11 from Internet Archive capture 20261005030805. Reasoning effort xhigh ('Models were run with maximum available reasoning effort in our API (xhigh for GPT-5.2 Thinking & Pro ...)'). GPT-5.2 Thinking = API gpt-5.2. SETTING DIFFERS: chart caption 'For the Telecom domain, we included a brief, generally helpful instruction in the system prompt to boost performance.' User model not stated. Vendor-reported (rule b).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "tau2-telecom",
    "score": 0.934,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano', 'Tool-calling' table (columns GPT-5.4 (xhigh) | GPT-5.4 mini (xhigh) | GPT-5.4 nano (xhigh) | GPT-5 mini (high)), row 'τ2-bench (telecom)', column 'GPT-5.4 mini (xhigh)', printed '93.4%'; read 2026-10-11 from Internet Archive capture 20260929161725 (openai.com returns HTTP 403). Reasoning effort xhigh. Prompt setting not stated on this page; the same table prints GPT-5.4 at 98.9% while OpenAI's GPT-5.5 post gives GPT-5.4 92.8% with 'original prompts', so this table used OpenAI's adjusted telecom prompt of that period. GPT-5.4's 98.9 and GPT-5 mini's 74.1 are not added (both models already have a telecom row in this OpenAI run). Vendor-reported (rule b).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "gpt-5-4-nano",
    "bench": "tau2-telecom",
    "score": 0.925,
    "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano",
    "by": "lab",
    "date": "2026-03-17",
    "note": "OpenAI 'Introducing GPT-5.4 mini and nano', 'Tool-calling' table (columns GPT-5.4 (xhigh) | GPT-5.4 mini (xhigh) | GPT-5.4 nano (xhigh) | GPT-5 mini (high)), row 'τ2-bench (telecom)', column 'GPT-5.4 nano (xhigh)', printed '92.5%'; read 2026-10-11 from Internet Archive capture 20260929161725 (openai.com returns HTTP 403). Reasoning effort xhigh. Prompt setting not stated on this page; the same table prints GPT-5.4 at 98.9% while OpenAI's GPT-5.5 post gives GPT-5.4 92.8% with 'original prompts', so this table used OpenAI's adjusted telecom prompt of that period. GPT-5.4's 98.9 and GPT-5 mini's 74.1 are not added (both models already have a telecom row in this OpenAI run). Vendor-reported (rule b).",
    "source_id": "first-party-openai-scores"
  },
  {
    "model": "qwen3-235b-a22b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.456,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=22",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 6 'Comparison among Qwen3-VL-235B-A22B (Thinking) and other reasoning baselines' on PDF page 22, row 'TAU2-Telecom', column 'Qwen3 235B-A22B Thinking-2507' = 45.6; re-read 2026-10-11 with pdftotext. Thinking mode. Dated 2507 thinking checkpoint; the same value is printed in the model's own card (reprint kept: this is Qwen's VL-report run, owner decision 2026-10-11 to keep correlated rows). Joins the existing first-party-qwen-scores tau2-telecom rows from this report.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.228,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'TAU2-Telecom', column 'Qwen3 30B-A3B Thinking' = 22.8; re-read 2026-10-11 with pdftotext. Thinking mode. Hybrid April-2025 Qwen/Qwen3-30B-A3B in thinking mode. Joins the existing first-party-qwen-scores tau2-telecom rows from this report.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "tau2-telecom",
    "score": 0.263,
    "source": "https://arxiv.org/pdf/2511.21631v2#page=23",
    "by": "lab",
    "date": "2025-11-27",
    "note": "Qwen3-VL Technical Report, arXiv 2511.21631v2 (arXiv non-exclusive licence -> rule b, Qwen's own models only), Table 8 'Comparison among Qwen3-VL-32B (Thinking), Qwen3-VL-30B-A3B (Thinking), and corresponding baselines' on PDF page 23, row 'TAU2-Telecom', column 'Qwen3 30B-A3B Thinking-2507' = 26.3; re-read 2026-10-11 with pdftotext. Thinking mode. Dated 2507 thinking checkpoint; the same value is printed in the model's own card (reprint kept: this is Qwen's VL-report run, owner decision 2026-10-11 to keep correlated rows). Joins the existing first-party-qwen-scores tau2-telecom rows from this report.",
    "source_id": "first-party-qwen-scores"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "tau2-telecom",
    "score": 0.968,
    "source": "https://mimo.xiaomi.com/mimo-v2-pro",
    "by": "lab",
    "date": "2026-03-18",
    "note": "Xiaomi MiMo-V2-Pro launch page (mimo.xiaomi.com/mimo-v2-pro, embedded in mimo.xiaomi.com/blog/mimo-v2-pro; dated 'March 18th, 2026'), benchmark chart, entry 'τ2-bench (Telecom)' (category 'Tool Use'), MiMo-V2-Pro = 96.8. The value is in the page HTML as chart data (benchData: {name: 'τ2-bench (Telecom)', scores: {mimo: 96.8, ...}}) and the page renders it as a text label on the bar (displayScore = score.toFixed(1)), the same treatment as the existing MiMo-V2-Omni / MiMo-V2.5 CharXiv rows of this ledger entry; printed chart data label (owner policy 2026-10-09). Other bars (MiMo-V2-Flash, Claude, Gemini, GPT) not used. Vendor-reported (rule b). Single-model run under this entry.",
    "source_id": "first-party-xiaomi-scores"
  },
  {
    "model": "dola-seed-2-0-pro",
    "bench": "tau2-telecom",
    "score": 0.942,
    "source": "https://arxiv.org/pdf/2607.00248v1",
    "by": "lab",
    "date": "2026-06-30",
    "note": "ByteDance Seed, 'Seed2.0 Model Card: Towards Intelligence Frontier for Real-World Complexity' (arXiv 2607.00248v1, CC BY 4.0; PDF only), Sec. 3.3 'Fundamental Agentic Capacity', agentic table comparing GPT-5.2 High | Claude-Sonnet-4.5 | Claude-Opus-4.5 | Gemini-3-pro High | Seed2.0 Pro, row 'τ2-Bench (telecom)', column 'Seed2.0 Pro' = 94.2; read 2026-10-11 with pdftotext. Own model (by: lab). Competitor cells not used: Sec. 3.3 'the final score is defined as the maximum of the score reported in the official documentation and the score obtained in our tests'. Identity: catalogue dola-seed-2-0-pro (Arena entry 'dola-seed-2.0-pro', the only catalogued Seed 2.0 Pro; 'Dola' is ByteDance's consumer-app brand prefix on the same Seed2.0 Pro name). No document ties the two explicitly, so this mapping is flagged for the owner.",
    "source_id": "paper-2607.00248v1-seed2-0"
  },
  {
    "model": "seed-2-0-lite",
    "bench": "tau2-telecom",
    "score": 0.921,
    "source": "https://arxiv.org/pdf/2607.00248v1",
    "by": "lab",
    "date": "2026-06-30",
    "note": "ByteDance Seed, 'Seed2.0 Model Card: Towards Intelligence Frontier for Real-World Complexity' (arXiv 2607.00248v1, CC BY 4.0; PDF only), Sec. 3.3 'Fundamental Agentic Capacity', agentic table comparing mini High | Flash High | Lite, row 'τ2-Bench (telecom)', column 'Lite' = 92.1; read 2026-10-11 with pdftotext. Own model (by: lab). Competitor cells not used: Sec. 3.3 'the final score is defined as the maximum of the score reported in the official documentation and the score obtained in our tests'.",
    "source_id": "paper-2607.00248v1-seed2-0"
  },
  {
    "model": "glm-4-7",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.333,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "GLM-4.7 model card @602d01e (MIT), 'Performances on Benchmarks' table, row 'Terminal Bench Hard', column 'GLM-4.7' = 33.3; re-read 2026-10-11. Card 'Evaluation Parameters' give Z.ai's own Terminal Bench settings (temperature 0.7, top-p 1.0, max new tokens 16,384); no third party is credited for this row. Caveat kept from pass 6: the row's other-lab cells (Kimi K2 Thinking 30.6, DeepSeek-V3.2 35.4, Gemini 3.0 Pro 39.0, Claude Sonnet 4.5 33.3, GPT-5-High 30.5, GPT-5.1-High 43.0) look like figures from an outside source and are not used. Terminal-Bench Hard = the 47-task hard subset of the original Terminal-Bench run with Terminus 2 (see the benchmark description proposal in pass6-tbhard).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "glm-4-6",
    "bench": "terminalbench-hard-gemma4",
    "score": 0.236,
    "source": "https://huggingface.co/zai-org/GLM-4.7/blob/602d01efcdd332c5238ca4bcede555defbe83eb7/README.md",
    "by": "lab",
    "date": "2025-12-22",
    "note": "GLM-4.7 model card @602d01e (MIT), 'Performances on Benchmarks' table, row 'Terminal Bench Hard', column 'GLM-4.6' = 23.6; re-read 2026-10-11. Card 'Evaluation Parameters' give Z.ai's own Terminal Bench settings (temperature 0.7, top-p 1.0, max new tokens 16,384); no third party is credited for this row. Same-lab sibling column (Z.ai's previous model). Caveat kept from pass 6: the row's other-lab cells (Kimi K2 Thinking 30.6, DeepSeek-V3.2 35.4, Gemini 3.0 Pro 39.0, Claude Sonnet 4.5 33.3, GPT-5-High 30.5, GPT-5.1-High 43.0) look like figures from an outside source and are not used. Terminal-Bench Hard = the 47-task hard subset of the original Terminal-Bench run with Terminus 2 (see the benchmark description proposal in pass6-tbhard).",
    "source_id": "card2-zai-org-glm-4-7"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "tau-bench-retail",
    "score": 0.062,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'Llama 3.3 70B Instruct', column 'Taubench Retail P@1' = 6.2; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint meta-llama/Llama-3.3-70B-Instruct. The very low retail P@1 is as printed.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-3-70b",
    "bench": "tau-bench-airline",
    "score": 0.353,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'Llama 3.3 70B Instruct', column 'Taubench Airline P@1' = 35.3; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint meta-llama/Llama-3.3-70B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "tau-bench-retail",
    "score": 0.291,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'Llama 3.1 405B Instruct', column 'Taubench Retail P@1' = 29.1; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "tau-bench-airline",
    "score": 0.26,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'Llama 3.1 405B Instruct', column 'Taubench Airline P@1' = 26.0; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint meta-llama/Llama-3.1-405B-Instruct.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "tau-bench-retail",
    "score": 0.548,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'DeepSeek V3', column 'Taubench Retail P@1' = 54.8; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint deepseek-ai/DeepSeek-V3 (catalogue deepseek-v3), the same mapping as this report's existing DeepSeek V3 rows (MBPP+, LiveCodeBench, BigCodeBench, ...).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "deepseek-v3",
    "bench": "tau-bench-airline",
    "score": 0.255,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 6 'Taubench Results', row 'DeepSeek V3', column 'Taubench Airline P@1' = 25.5; re-read 2026-10-11 (pdftotext). Caption: 'We follow the original experimental setup from Yao et al. (2024). Pass@k (P@k) evaluates a model's consistency ... Scores are aggregated over 10 runs.' Provenance: Sec. 4 'Wherever possible, we show externally reported results ... Where these are not available, we attempt to internally reproduce these results'; Pass^1-4 over 10 runs for this model is published nowhere else, so the row is Cohere's own reproduction (P@1 comes from the same 10 runs). Checkpoint deepseek-ai/DeepSeek-V3 (catalogue deepseek-v3), the same mapping as this report's existing DeepSeek V3 rows (MBPP+, LiveCodeBench, BigCodeBench, ...).",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "llama-3-1-405b",
    "bench": "aime-2024",
    "score": 0.2,
    "source": "https://arxiv.org/pdf/2504.00698v2",
    "by": "independent",
    "date": "2025-04-14",
    "note": "Command A technical report (Cohere, arXiv 2504.00698v2, CC BY 4.0), Table 15, row 'Llama 3.3 405B', column 'AIME (2024)' = 20.0* ; re-read 2026-10-11. Caption: 'Results for external models are taken from officially-reported sources, unless indicated with an asterisk (*), which denotes internal evaluation since official public results were not available' (Cohere's own run). Identity: Meta released Llama 3.3 only as a 70B model; the report's other tables call the same model 'Llama 3.1 405B Instruct', and its code-table sources cite the Llama 3.3 model card, which lists Llama 3.1 405B as the comparator. So the row label is a slip for Llama 3.1 405B Instruct (catalogue llama-3-1-405b). MATH / GPQA cells of this row are unstarred (officially reported) and not used.",
    "source_id": "report-2504.00698v2"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "charxiv-rq",
    "score": 0.801,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-multimodal-bench.png (sha256 8abf8b5c…7b331), group 'CharXiv RQ', bar 'MiMo-V2-Omni' with printed data label '80.1'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "mmmu-pro",
    "score": 0.768,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-multimodal-bench.png (sha256 8abf8b5c…7b331), group 'MMMU-Pro', bar 'MiMo-V2-Omni' with printed data label '76.8'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "hr-bench-4k",
    "score": 0.833,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-multimodal-bench.png (sha256 8abf8b5c…7b331), group 'HR-Bench (4k)', bar 'MiMo-V2-Omni' with printed data label '83.3'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "dailyomni",
    "score": 0.805,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-multimodal-bench.png (sha256 8abf8b5c…7b331), group 'DailyOmni', bar 'MiMo-V2-Omni' with printed data label '80.5'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-omni",
    "bench": "videoholmes",
    "score": 0.595,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-multimodal-bench.png (sha256 8abf8b5c…7b331), group 'VideoHolmes', bar 'MiMo-V2-Omni' with printed data label '59.5'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "terminal-bench-2",
    "score": 0.571,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-coding-bench.png (sha256 a6475762…e4c3d), group 'Terminal-Bench 2.0', bar 'MiMo-V2-Pro' with printed data label '57.1'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.55,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5 model card @6365158, chart assets/mimo-v2.5-coding-bench.png (sha256 a6475762…e4c3d), group 'SWE-Bench Pro', bar 'MiMo-V2-Pro' with printed data label '55.0'; read 2026-10-11 (printed label, not a bar height; owner policy 2026-10-09). Same-lab sibling in Xiaomi's own chart, same settings as the existing MiMo-V2.5 row of this group (settings not stated in the card). Other labs' bars not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5"
  },
  {
    "model": "glm-4-6",
    "bench": "aime-2025",
    "score": 0.939,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'AIME 25', bar GLM-4.6 (label inside the bar; 98.6 above it is the w/ Tools value), printed data label 93.9; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "gpqa-unspecified",
    "score": 0.81,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'GPQA', bar GLM-4.6 (label inside the bar; 82.9 w/ Tools above it), printed data label 81.0; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.828,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'LiveCodeBench v6', bar GLM-4.6 (label inside the bar; 84.5 w/ Tools above it), printed data label 82.8; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "hle",
    "score": 0.172,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'HLE', bar GLM-4.6 (label inside the bar), printed data label 17.2; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "hle-with-tools",
    "score": 0.304,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'HLE', bar GLM-4.6 (30.4 'w/ Tools' above the bar), printed data label 30.4; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "browsecomp",
    "score": 0.451,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'BrowseComp', bar GLM-4.6, printed data label 45.1; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-6",
    "bench": "swe-bench-verified",
    "score": 0.68,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'SWE-bench Verified', bar GLM-4.6, printed data label 68.0; read 2026-10-11 (printed label, owner policy 2026-10-09). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-5",
    "bench": "terminal-bench-pre-2",
    "score": 0.375,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'Terminal-Bench', bar GLM-4.5, printed data label 37.5; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab GLM-4.5 bar (Z.ai's previous model). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-5",
    "bench": "gpqa-unspecified",
    "score": 0.799,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'GPQA', bar GLM-4.5, printed data label 79.9; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab GLM-4.5 bar (Z.ai's previous model). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-5",
    "bench": "hle",
    "score": 0.144,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'HLE', bar GLM-4.5, printed data label 14.4; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab GLM-4.5 bar (Z.ai's previous model). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-5",
    "bench": "browsecomp",
    "score": 0.264,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'BrowseComp', bar GLM-4.5, printed data label 26.4; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab GLM-4.5 bar (Z.ai's previous model). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "glm-4-5",
    "bench": "swe-bench-verified",
    "score": 0.642,
    "source": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md",
    "by": "lab",
    "date": "2025-09-30",
    "note": "Z.ai's own chart embedded by the pinned GLM-4.6 README ('LLM Performance Evaluation: Agentic, Reasoning and Coding'; resources/bench_glm46.png at zai-org/GLM-4.5 commit 6fb1bbf, sha256 fe12649f…07c0c46; 'Evaluation results under 128K context length'), group 'SWE-bench Verified', bar GLM-4.5, printed data label 64.2; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab GLM-4.5 bar (Z.ai's previous model). DeepSeek and Claude bars not used. Earlier imports skipped these cells because the models already had the figures from other sources (e.g. the GLM-4.7 card); recorded here as Z.ai's GLM-4.6-launch run (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card2-zai-org-glm-4-6"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "minif2f-pass1",
    "score": 0.0572,
    "source": "https://arxiv.org/html/2512.20848v1",
    "by": "independent",
    "date": "2025-12-23",
    "note": "NVIDIA Nemotron 3 Nano report (arXiv 2512.20848v1, CC BY 4.0), Table 3 'Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B', row 'MiniF2F pass@1', column 'Qwen3' = 5.72* ; re-read 2026-10-11. Provenance: the report's rule is 'we use the officially reported numbers whenever available; if a benchmark is not reported, we take the value from ArtificialAnalysis (AA); and if neither source provides results, we may compute the scores ourselves using the official evaluation protocol'. Neither maker nor AA publishes MiniF2F pass@1 for this model, and the starred two-decimal cell differs from Qwen's own MMLU-ProX figure, so the '*' marks NVIDIA's own computation. Checkpoint per caption: Qwen3-30B-A3B-Thinking-2507.",
    "source_id": "report-2512.20848v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "minif2f-pass1",
    "score": 0.1205,
    "source": "https://arxiv.org/html/2512.20848v1",
    "by": "independent",
    "date": "2025-12-23",
    "note": "NVIDIA Nemotron 3 Nano report (arXiv 2512.20848v1, CC BY 4.0), Table 3 'Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B', row 'MiniF2F pass@1', column 'GPT-OSS' = 12.05* ; re-read 2026-10-11. Provenance: the report's rule is 'we use the officially reported numbers whenever available; if a benchmark is not reported, we take the value from ArtificialAnalysis (AA); and if neither source provides results, we may compute the scores ourselves using the official evaluation protocol'. Neither maker nor AA publishes MiniF2F pass@1 for this model, and the starred two-decimal cell differs from Qwen's own MMLU-ProX figure, so the '*' marks NVIDIA's own computation. Checkpoint per caption: OpenAI gpt-oss-20b (reasoning effort not printed).",
    "source_id": "report-2512.20848v1"
  },
  {
    "model": "qwen3-30b-a3b-thinking-2507",
    "bench": "mmlu-prox",
    "score": 0.776,
    "source": "https://arxiv.org/html/2512.20848v1",
    "by": "independent",
    "date": "2025-12-23",
    "note": "NVIDIA Nemotron 3 Nano report (arXiv 2512.20848v1, CC BY 4.0), Table 3 'Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B', row 'MMLU-ProX (avg over langs)', column 'Qwen3' = 77.60* ; re-read 2026-10-11. Provenance: the report's rule is 'we use the officially reported numbers whenever available; if a benchmark is not reported, we take the value from ArtificialAnalysis (AA); and if neither source provides results, we may compute the scores ourselves using the official evaluation protocol'. Neither maker nor AA publishes MMLU-ProX (avg over langs) for this model, and the starred two-decimal cell differs from Qwen's own MMLU-ProX figure, so the '*' marks NVIDIA's own computation. Checkpoint per caption: Qwen3-30B-A3B-Thinking-2507.",
    "source_id": "report-2512.20848v1"
  },
  {
    "model": "gpt-oss-20b",
    "bench": "mmlu-prox",
    "score": 0.691,
    "source": "https://arxiv.org/html/2512.20848v1",
    "by": "independent",
    "date": "2025-12-23",
    "note": "NVIDIA Nemotron 3 Nano report (arXiv 2512.20848v1, CC BY 4.0), Table 3 'Nemotron 3 Nano compared to Qwen3-30B-A3B-Thinking-2507, and GPT-OSS 20B', row 'MMLU-ProX (avg over langs)', column 'GPT-OSS' = 69.10* ; re-read 2026-10-11. Provenance: the report's rule is 'we use the officially reported numbers whenever available; if a benchmark is not reported, we take the value from ArtificialAnalysis (AA); and if neither source provides results, we may compute the scores ourselves using the official evaluation protocol'. Neither maker nor AA publishes MMLU-ProX (avg over langs) for this model, and the starred two-decimal cell differs from Qwen's own MMLU-ProX figure, so the '*' marks NVIDIA's own computation. Checkpoint per caption: OpenAI gpt-oss-20b (reasoning effort not printed).",
    "source_id": "report-2512.20848v1"
  },
  {
    "model": "openthinker3-7b",
    "bench": "math-subset-unspecified",
    "score": 0.945,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'MATH', column 'OpenThinker3-7B' = 94.5; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: open-thoughts/OpenThinker3-7B. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "openthinker3-7b",
    "bench": "omega",
    "score": 0.384,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'OpenThinker3-7B' = 38.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: open-thoughts/OpenThinker3-7B. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "math-subset-unspecified",
    "score": 0.944,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Nemotron-Nano-9B-v2' = 94.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: nvidia/NVIDIA-Nemotron-Nano-9B-v2. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "nemotron-nano-9b-v2",
    "bench": "omega",
    "score": 0.424,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Nemotron-Nano-9B-v2' = 42.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: nvidia/NVIDIA-Nemotron-Nano-9B-v2. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "math-subset-unspecified",
    "score": 0.879,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'MATH', column 'DeepSeek-R1-Distill-Qwen-7B' = 87.9; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "deepseek-r1-distill-qwen-7b",
    "bench": "omega",
    "score": 0.285,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'DeepSeek-R1-Distill-Qwen-7B' = 28.5; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "qwen3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.951,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Qwen 3 8B (reasoning)' = 95.1; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: Qwen/Qwen3-8B in thinking mode. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "qwen3-8b",
    "bench": "omega",
    "score": 0.434,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Qwen 3 8B (reasoning)' = 43.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: Qwen/Qwen3-8B in thinking mode. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "math-subset-unspecified",
    "score": 0.952,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Qwen 3 VL 8B Thinker' = 95.2; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: Qwen/Qwen3-VL-8B-Thinking. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "qwen3-vl-8b-thinking",
    "bench": "omega",
    "score": 0.381,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Think/blob/d97e442d7cc678210054dbcc9b440894d62c89a4/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Think model card @d97e442 (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Qwen 3 VL 8B Thinker' = 38.1; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0), Table 15 (same numbers): 'We evaluate all models using our evaluation framework' and Sec. 4.1.1 'We establish a standard evaluation configuration between all baseline models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95'. Checkpoint: Qwen/Qwen3-VL-8B-Thinking. MATH subset not stated.",
    "source_id": "card2-allenai-olmo-3-7b-think"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'MMLU-Pro', column 'Qwen3-30B-A3B Non-Thinking' = 69.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-pro",
    "score": 0.58,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'MMLU-Pro', column 'Qwen3-4B Non-Thinking' = 58.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-redux",
    "score": 0.841,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-30B-A3B Non-Thinking' = 84.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-redux",
    "score": 0.773,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-4B Non-Thinking' = 77.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "gpqa-unspecified",
    "score": 0.548,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-30B-A3B Non-Thinking' = 54.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "gpqa-unspecified",
    "score": 0.417,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-4B Non-Thinking' = 41.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "supergpqa",
    "score": 0.422,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-30B-A3B Non-Thinking' = 42.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "supergpqa",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-4B Non-Thinking' = 32.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aime-2025",
    "score": 0.216,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'AIME25', column 'Qwen3-30B-A3B Non-Thinking' = 21.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "aime-2025",
    "score": 0.191,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'AIME25', column 'Qwen3-4B Non-Thinking' = 19.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.12,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-30B-A3B Non-Thinking' = 12.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.121,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-4B Non-Thinking' = 12.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "zebralogic",
    "score": 0.332,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'ZebraLogic', column 'Qwen3-30B-A3B Non-Thinking' = 33.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "zebralogic",
    "score": 0.352,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'ZebraLogic', column 'Qwen3-4B Non-Thinking' = 35.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livebench-20241125",
    "score": 0.594,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-30B-A3B Non-Thinking' = 59.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "livebench-20241125",
    "score": 0.484,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-4B Non-Thinking' = 48.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.29,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-30B-A3B Non-Thinking' = 29.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "livecodebench-v6",
    "score": 0.264,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-4B Non-Thinking' = 26.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multipl-e",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-30B-A3B Non-Thinking' = 74.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "multipl-e",
    "score": 0.666,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-4B Non-Thinking' = 66.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aider-polyglot",
    "score": 0.244,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-30B-A3B Non-Thinking' = **24.4**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "aider-polyglot",
    "score": 0.138,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-4B Non-Thinking' = 13.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-30B-A3B Non-Thinking' = **83.7**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "ifeval-unspecified",
    "score": 0.812,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-4B Non-Thinking' = 81.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.248,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-30B-A3B Non-Thinking' = 24.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "arena-hard-v2",
    "score": 0.095,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-4B Non-Thinking' = 9.5; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "creative-writing-v3",
    "score": 0.681,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-30B-A3B Non-Thinking' = 68.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "creative-writing-v3",
    "score": 0.536,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-4B Non-Thinking' = 53.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "writingbench",
    "score": 0.722,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-30B-A3B Non-Thinking' = 72.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "writingbench",
    "score": 0.685,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-4B Non-Thinking' = 68.5; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.586,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-30B-A3B Non-Thinking' = 58.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "bfcl-v3",
    "score": 0.576,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-4B Non-Thinking' = 57.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-retail",
    "score": 0.383,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU1-Retail', column 'Qwen3-30B-A3B Non-Thinking' = 38.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau-bench-retail",
    "score": 0.243,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU1-Retail', column 'Qwen3-4B Non-Thinking' = 24.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-airline",
    "score": 0.18,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU1-Airline', column 'Qwen3-30B-A3B Non-Thinking' = 18.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau-bench-airline",
    "score": 0.16,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU1-Airline', column 'Qwen3-4B Non-Thinking' = 16.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-retail",
    "score": 0.316,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU2-Retail', column 'Qwen3-30B-A3B Non-Thinking' = 31.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-retail",
    "score": 0.281,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU2-Retail', column 'Qwen3-4B Non-Thinking' = 28.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-airline",
    "score": 0.18,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU2-Airline', column 'Qwen3-30B-A3B Non-Thinking' = 18.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "tau2-airline",
    "score": 0.12,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Agent', row 'TAU2-Airline', column 'Qwen3-4B Non-Thinking' = 12.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multi-if-unspecified",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-30B-A3B Non-Thinking' = **70.8**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "multi-if-unspecified",
    "score": 0.613,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-4B Non-Thinking' = 61.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.651,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-30B-A3B Non-Thinking' = **65.1**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "mmlu-prox",
    "score": 0.496,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-4B Non-Thinking' = 49.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "include",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-30B-A3B Non-Thinking' = **67.8**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "include",
    "score": 0.538,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-4B Non-Thinking' = 53.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "polymath",
    "score": 0.233,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-30B-A3B Non-Thinking' = 23.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "qwen3-4b",
    "bench": "polymath",
    "score": 0.166,
    "source": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md",
    "by": "lab",
    "date": "2025-08-06",
    "note": "Qwen's own evaluation, pinned Qwen3-4B-Instruct-2507 (@cdbee75) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-4B Non-Thinking' = 16.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab hybrid column in Qwen's own table (catalogue qwen3-30b-a3b / qwen3-4b = the April 2025 hybrid checkpoints). Held earlier only because hybrid models were recorded in thinking mode only; that convention was already dropped for the TAU2-Telecom cells of this card and for the Qwen3-235B-A22B-Instruct-2507 card.",
    "source_id": "card-qwen--qwen3-4b-instruct-2507"
  },
  {
    "model": "o4-mini",
    "bench": "mmlu-pro",
    "score": 0.819,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-Pro', column 'OpenAI O4-mini' = 81.9; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "mmlu-pro",
    "score": 0.859,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-Pro', column 'OpenAI O3' = **85.9**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "mmlu-redux",
    "score": 0.928,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-Redux', column 'OpenAI O4-mini' = 92.8; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "mmlu-redux",
    "score": 0.949,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-Redux', column 'OpenAI O3' = **94.9**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "supergpqa",
    "score": 0.564,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'SuperGPQA', column 'OpenAI O4-mini' = 56.4; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.667,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'HMMT25', column 'OpenAI O4-mini' = 66.7; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.775,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'HMMT25', column 'OpenAI O3' = 77.5; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "livecodebench-v6",
    "score": 0.718,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'LiveCodeBench v6 (25.02-25.05)', column 'OpenAI O4-mini' = 71.8; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "livecodebench-v6",
    "score": 0.586,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'LiveCodeBench v6 (25.02-25.05)', column 'OpenAI O3' = 58.6; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "ojbench",
    "score": 0.333,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'OJBench', column 'OpenAI O4-mini' = 33.3; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "ojbench",
    "score": 0.254,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'OJBench', column 'OpenAI O3' = 25.4; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "ifeval-unspecified",
    "score": 0.924,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'IFEval', column 'OpenAI O4-mini' = **92.4**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "ifeval-unspecified",
    "score": 0.921,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'IFEval', column 'OpenAI O3' = 92.1; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "arena-hard-v2",
    "score": 0.593,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'Arena-Hard v2$', column 'OpenAI O4-mini' = 59.3; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Arena-Hard v2 win rate judged by GPT-4.1 (card footnote '$'). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "arena-hard-v2",
    "score": 0.808,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'Arena-Hard v2$', column 'OpenAI O3' = **80.8**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Arena-Hard v2 win rate judged by GPT-4.1 (card footnote '$'). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "creative-writing-v3",
    "score": 0.788,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'Creative Writing v3', column 'OpenAI O4-mini' = 78.8; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "creative-writing-v3",
    "score": 0.877,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'Creative Writing v3', column 'OpenAI O3' = **87.7**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "writingbench",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'WritingBench', column 'OpenAI O4-mini' = 78.4; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "writingbench",
    "score": 0.853,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'WritingBench', column 'OpenAI O3' = 85.3; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-retail",
    "score": 0.71,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Retail', column 'OpenAI O4-mini' = 71.0; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "tau2-retail",
    "score": 0.763,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Retail', column 'OpenAI O3' = **76.3**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-airline",
    "score": 0.59,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Airline', column 'OpenAI O4-mini' = 59.0; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "tau2-airline",
    "score": 0.7,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Airline', column 'OpenAI O3' = **70.0**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "tau2-telecom",
    "score": 0.42,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Telecom', column 'OpenAI O4-mini' = 42.0; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "tau2-telecom",
    "score": 0.605,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'TAU2-Telecom', column 'OpenAI O3' = **60.5**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "multi-if-unspecified",
    "score": 0.78,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MultiIF', column 'OpenAI O4-mini' = 78.0; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "multi-if-unspecified",
    "score": 0.803,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MultiIF', column 'OpenAI O3' = 80.3; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "mmlu-prox",
    "score": 0.79,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-ProX', column 'OpenAI O4-mini' = 79.0; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "mmlu-prox",
    "score": 0.833,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'MMLU-ProX', column 'OpenAI O3' = 83.3; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "include",
    "score": 0.808,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'INCLUDE', column 'OpenAI O4-mini' = 80.8; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "include",
    "score": 0.866,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'INCLUDE', column 'OpenAI O3' = **86.6**; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o4-mini",
    "bench": "polymath",
    "score": 0.487,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'PolyMATH', column 'OpenAI O4-mini' = 48.7; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o4-mini (catalogue o4-mini).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3",
    "bench": "polymath",
    "score": 0.497,
    "source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Qwen3-235B-A22B-Thinking-2507 model card @6cbffae, Performance table, row 'PolyMATH', column 'OpenAI O3' = 49.7; percent converted to fraction; re-read 2026-10-11. Qwen's own run at medium reasoning effort: card footnote '* For OpenAI O4-mini and O3, we use a medium reasoning effort, except for scores marked with *, which are generated using high reasoning effort.' Only unstarred cells on rows where OpenAI published no o3/o4-mini figure are used (starred cells, and the GPQA/AIME25/HLE/TAU1 rows, may be OpenAI's own high-effort numbers; LiveBench and BFCL rows skipped because public leaderboards list these models). Setting: reasoning effort medium (not the model's highest). Checkpoint: OpenAI o3 (catalogue o3).",
    "source_id": "card-qwen--qwen3-235b-a22b-thinking-2507"
  },
  {
    "model": "o3-mini",
    "bench": "flenqa-3k",
    "score": 0.968,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'FlenQA [3K-token subset]', column 'o3-mini' = 96.8; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: o3-mini 2025-01-31 (high) per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "flenqa-3k",
    "score": 0.908,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'FlenQA [3K-token subset]', column 'GPT-4o' = 90.8; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: GPT-4o 2024-05-13 per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "ifeval-unspecified",
    "score": 0.818,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'IFEval Strict', column 'GPT-4o' = 81.8; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: GPT-4o 2024-05-13 per Table 3. Filed under ifeval-unspecified, the id the existing IFEval Strict rows of this table use.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4",
    "bench": "arena-hard-unspecified",
    "score": 0.681,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'ArenaHard', column 'Phi-4' = 68.1⋆; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0; '⋆' marks a discrepancy with the Phi-4 paper 'due to changing the backend LLM-judge', i.e. a re-run). Own model (by: lab).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning",
    "bench": "arena-hard-unspecified",
    "score": 0.733,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'ArenaHard', column 'Phi-4-reasoning' = 73.3; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Own model (by: lab).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "phi-4-reasoning-plus",
    "bench": "arena-hard-unspecified",
    "score": 0.79,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "lab",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'ArenaHard', column 'Phi-4-reasoning-plus' = 79.0; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Own model (by: lab).",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "arena-hard-unspecified",
    "score": 0.819,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'ArenaHard', column 'o3-mini' = 81.9; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: o3-mini 2025-01-31 (high) per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "arena-hard-unspecified",
    "score": 0.69,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'ArenaHard', column 'GPT-4o' = 69.0⋆; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0; '⋆' marks a discrepancy with the Phi-4 paper 'due to changing the backend LLM-judge', i.e. a re-run). Checkpoint: GPT-4o 2024-05-13 per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "humaneval-plus",
    "score": 0.94,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'HumanEvalPlus', column 'o3-mini' = 94.0; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: o3-mini 2025-01-31 (high) per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "humaneval-plus",
    "score": 0.849,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'HumanEvalPlus', column 'GPT-4o' = 84.9; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: GPT-4o 2024-05-13 per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "o3-mini",
    "bench": "mmlu-pro",
    "score": 0.794,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'MMLUPro', column 'o3-mini' = 79.4; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: o3-mini 2025-01-31 (high) per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "gpt-4o-2024-05-13",
    "bench": "mmlu-pro",
    "score": 0.735,
    "source": "https://arxiv.org/html/2504.21318v1",
    "by": "independent",
    "date": "2025-04-30",
    "note": "Phi-4-reasoning technical report (Microsoft, arXiv 2504.21318v1, CC BY 4.0), Table 2 'Average pass@1 accuracy of models across general-purpose benchmarks evaluated averaged over five generations', row 'MMLUPro', column 'GPT-4o' = 73.5; read 2026-10-11. Microsoft's own runs (Sec. 5: 'we evaluate our models on MMLU-pro, HumanEvalPlus, and ArenaHard ... general-purpose benchmarks from the Eureka ML Insights repository, namely FlenQA ... and IFEval'; caption: Phi-4-reasoning(-plus) at temperature 0.8, Phi-4 at 0.0). Checkpoint: GPT-4o 2024-05-13 per Table 3.",
    "source_id": "report-2504.21318v1"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-pro",
    "score": 0.752,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'MMLU-Pro', column 'Qwen3-235B-A22B Non-Thinking' = 75.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-pro",
    "score": 0.691,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'MMLU-Pro', column 'Qwen3-30B-A3B Non-Thinking' = 69.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-redux",
    "score": 0.892,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-235B-A22B Non-Thinking' = 89.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-redux",
    "score": 0.841,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-30B-A3B Non-Thinking' = 84.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-redux",
    "score": 0.893,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'MMLU-Redux', column 'Qwen3-30B-A3B-Instruct-2507' = 89.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "gpqa-unspecified",
    "score": 0.629,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-235B-A22B Non-Thinking' = 62.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "gpqa-unspecified",
    "score": 0.548,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-30B-A3B Non-Thinking' = 54.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "gpqa-unspecified",
    "score": 0.704,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'GPQA', column 'Qwen3-30B-A3B-Instruct-2507' = 70.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "supergpqa",
    "score": 0.482,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-235B-A22B Non-Thinking' = 48.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "supergpqa",
    "score": 0.422,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-30B-A3B Non-Thinking' = 42.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "supergpqa",
    "score": 0.534,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Knowledge', row 'SuperGPQA', column 'Qwen3-30B-A3B-Instruct-2507' = 53.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aime-2025",
    "score": 0.247,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'AIME25', column 'Qwen3-235B-A22B Non-Thinking' = 24.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aime-2025",
    "score": 0.216,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'AIME25', column 'Qwen3-30B-A3B Non-Thinking' = 21.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.1,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-235B-A22B Non-Thinking' = 10.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.12,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-30B-A3B Non-Thinking' = 12.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.43,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'HMMT25', column 'Qwen3-30B-A3B-Instruct-2507' = 43.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "zebralogic",
    "score": 0.377,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'ZebraLogic', column 'Qwen3-235B-A22B Non-Thinking' = 37.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "zebralogic",
    "score": 0.332,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'ZebraLogic', column 'Qwen3-30B-A3B Non-Thinking' = 33.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livebench-20241125",
    "score": 0.625,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-235B-A22B Non-Thinking' = 62.5; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livebench-20241125",
    "score": 0.594,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-30B-A3B Non-Thinking' = 59.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "livebench-20241125",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Reasoning', row 'LiveBench 20241125', column 'Qwen3-30B-A3B-Instruct-2507' = 69.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "livecodebench-v6",
    "score": 0.329,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-235B-A22B Non-Thinking' = 32.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "livecodebench-v6",
    "score": 0.29,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'LiveCodeBench v6 (25.02-25.05)', column 'Qwen3-30B-A3B Non-Thinking' = 29.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "multipl-e",
    "score": 0.793,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-235B-A22B Non-Thinking' = 79.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multipl-e",
    "score": 0.746,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-30B-A3B Non-Thinking' = 74.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "multipl-e",
    "score": 0.838,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'MultiPL-E', column 'Qwen3-30B-A3B-Instruct-2507' = **83.8**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "aider-polyglot",
    "score": 0.596,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-235B-A22B Non-Thinking' = **59.6**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "aider-polyglot",
    "score": 0.244,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-30B-A3B Non-Thinking' = 24.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "aider-polyglot",
    "score": 0.356,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Coding', row 'Aider-Polyglot', column 'Qwen3-30B-A3B-Instruct-2507' = 35.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "ifeval-unspecified",
    "score": 0.832,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-235B-A22B Non-Thinking' = 83.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "ifeval-unspecified",
    "score": 0.837,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-30B-A3B Non-Thinking' = 83.7; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "ifeval-unspecified",
    "score": 0.847,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'IFEval', column 'Qwen3-30B-A3B-Instruct-2507' = **84.7**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "arena-hard-v2",
    "score": 0.52,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-235B-A22B Non-Thinking' = 52.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "arena-hard-v2",
    "score": 0.248,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-30B-A3B Non-Thinking' = 24.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "arena-hard-v2",
    "score": 0.69,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Arena-Hard v2*', column 'Qwen3-30B-A3B-Instruct-2507' = **69.0**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them. Card footnote: '*: For reproducibility, we report the win rates evaluated by GPT-4.1.'",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "creative-writing-v3",
    "score": 0.804,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-235B-A22B Non-Thinking' = 80.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "creative-writing-v3",
    "score": 0.681,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-30B-A3B Non-Thinking' = 68.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "creative-writing-v3",
    "score": 0.86,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'Creative Writing v3', column 'Qwen3-30B-A3B-Instruct-2507' = **86.0**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "writingbench",
    "score": 0.77,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-235B-A22B Non-Thinking' = 77.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "writingbench",
    "score": 0.722,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-30B-A3B Non-Thinking' = 72.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "writingbench",
    "score": 0.855,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Alignment', row 'WritingBench', column 'Qwen3-30B-A3B-Instruct-2507' = **85.5**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "bfcl-v3",
    "score": 0.68,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-235B-A22B Non-Thinking' = **68.0**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "bfcl-v3",
    "score": 0.586,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-30B-A3B Non-Thinking' = 58.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "bfcl-v3",
    "score": 0.651,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'BFCL-v3', column 'Qwen3-30B-A3B-Instruct-2507' = 65.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-retail",
    "score": 0.652,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU1-Retail', column 'Qwen3-235B-A22B Non-Thinking' = 65.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-retail",
    "score": 0.383,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU1-Retail', column 'Qwen3-30B-A3B Non-Thinking' = 38.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau-bench-airline",
    "score": 0.32,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU1-Airline', column 'Qwen3-235B-A22B Non-Thinking' = 32.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau-bench-airline",
    "score": 0.18,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU1-Airline', column 'Qwen3-30B-A3B Non-Thinking' = 18.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-retail",
    "score": 0.649,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Retail', column 'Qwen3-235B-A22B Non-Thinking' = 64.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-retail",
    "score": 0.316,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Retail', column 'Qwen3-30B-A3B Non-Thinking' = 31.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-airline",
    "score": 0.36,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Airline', column 'Qwen3-235B-A22B Non-Thinking' = 36.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-airline",
    "score": 0.18,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Airline', column 'Qwen3-30B-A3B Non-Thinking' = 18.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "tau2-telecom",
    "score": 0.246,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Telecom', column 'Qwen3-235B-A22B Non-Thinking' = 24.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "tau2-telecom",
    "score": 0.184,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Agent', row 'TAU2-Telecom', column 'Qwen3-30B-A3B Non-Thinking' = 18.4; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "multi-if-unspecified",
    "score": 0.702,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-235B-A22B Non-Thinking' = 70.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "multi-if-unspecified",
    "score": 0.708,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-30B-A3B Non-Thinking' = **70.8**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "multi-if-unspecified",
    "score": 0.679,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MultiIF', column 'Qwen3-30B-A3B-Instruct-2507' = 67.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "mmlu-prox",
    "score": 0.732,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-235B-A22B Non-Thinking' = 73.2; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "mmlu-prox",
    "score": 0.651,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-30B-A3B Non-Thinking' = 65.1; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "mmlu-prox",
    "score": 0.72,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'MMLU-ProX', column 'Qwen3-30B-A3B-Instruct-2507' = 72.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "include",
    "score": 0.756,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-235B-A22B Non-Thinking' = 75.6; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "include",
    "score": 0.678,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-30B-A3B Non-Thinking' = 67.8; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "include",
    "score": 0.719,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'INCLUDE', column 'Qwen3-30B-A3B-Instruct-2507' = 71.9; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-235b-a22b",
    "bench": "polymath",
    "score": 0.27,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-235B-A22B Non-Thinking' = 27.0; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b",
    "bench": "polymath",
    "score": 0.233,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-30B-A3B Non-Thinking' = 23.3; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (hybrid checkpoint run with thinking disabled; the only setting this source reports for it). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-30b-a3b-instruct-2507",
    "bench": "polymath",
    "score": 0.431,
    "source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507/blob/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe/README.md",
    "by": "lab",
    "date": "2025-07-29",
    "note": "Qwen's own evaluation, pinned Qwen3-30B-A3B-Instruct-2507 (@0d7cf23) model card Performance table, section 'Multilingualism', row 'PolyMATH', column 'Qwen3-30B-A3B-Instruct-2507' = **43.1**; percent converted to fraction; re-read 2026-10-11. Mode: non-thinking (instruct-only checkpoint). Same-lab column in Qwen's own table: hybrid columns map to the April 2025 hybrid checkpoints (catalogue qwen3-235b-a22b / qwen3-30b-a3b); the Instruct-2507 column is the card's own model (rows not imported earlier). Other makers' columns (DeepSeek-V3-0324, GPT-4o-0327 unmarked cells, Gemini-2.5-Flash) not used: no statement that Qwen ran them.",
    "source_id": "card-qwen--qwen3-30b-a3b-instruct-2507"
  },
  {
    "model": "qwen3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.823,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Qwen 3 8B (no reasoning)' = 82.3; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: hybrid Qwen/Qwen3-8B with thinking off (the only setting this source reports).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-8b",
    "bench": "omega",
    "score": 0.205,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Qwen 3 8B (no reasoning)' = 20.5; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: hybrid Qwen/Qwen3-8B with thinking off (the only setting this source reports).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-8b",
    "bench": "bfcl-version-unspecified",
    "score": 0.602,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'BFCL', column 'Qwen 3 8B (no reasoning)' = 60.2; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: hybrid Qwen/Qwen3-8B with thinking off (the only setting this source reports). BFCL version not stated (section 'Tool Use').",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "math-subset-unspecified",
    "score": 0.916,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Qwen 3 VL 8B Instruct' = 91.6; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-VL-8B-Instruct.",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "omega",
    "score": 0.323,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Qwen 3 VL 8B Instruct' = 32.3; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-VL-8B-Instruct.",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-vl-8b-instruct",
    "bench": "bfcl-version-unspecified",
    "score": 0.662,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'BFCL', column 'Qwen 3 VL 8B Instruct' = 66.2; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-VL-8B-Instruct. BFCL version not stated (section 'Tool Use').",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "math-subset-unspecified",
    "score": 0.71,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Qwen 2.5 7B' = 71.0; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen2.5-7B-Instruct (instruct-model table).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "omega",
    "score": 0.137,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Qwen 2.5 7B' = 13.7; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen2.5-7B-Instruct (instruct-model table).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen2-5-7b",
    "bench": "bfcl-version-unspecified",
    "score": 0.558,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'BFCL', column 'Qwen 2.5 7B' = 55.8; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen2.5-7B-Instruct (instruct-model table). BFCL version not stated (section 'Tool Use').",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "apertus-8b-instruct-2509",
    "bench": "math-subset-unspecified",
    "score": 0.219,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Apertus 8B Instruct' = 21.9; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: swiss-ai/Apertus-8B-Instruct-2509 (the only Apertus 8B Instruct before this Nov 2025 card).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "apertus-8b-instruct-2509",
    "bench": "omega",
    "score": 0.05,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Apertus 8B Instruct' = 5.0; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: swiss-ai/Apertus-8B-Instruct-2509 (the only Apertus 8B Instruct before this Nov 2025 card).",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "math-subset-unspecified",
    "score": 0.673,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'MATH', column 'Granite 3.3 8B Instruct' = 67.3; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: ibm-granite/granite-3.3-8b-instruct.",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "granite-3-3-8b",
    "bench": "omega",
    "score": 0.107,
    "source": "https://huggingface.co/allenai/Olmo-3-7B-Instruct/blob/6e5971d9eba42665f5bd5a0fcf047f299ce1dccc/README.md",
    "by": "independent",
    "date": "2026-06-25",
    "note": "Olmo-3-7B-Instruct model card @6e5971d (Apache-2.0), 'Evaluation' table, row 'OMEGA', column 'Granite 3.3 8B Instruct' = 10.7; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: ibm-granite/granite-3.3-8b-instruct.",
    "source_id": "card2-allenai-olmo-3-7b-instruct"
  },
  {
    "model": "qwen3-32b",
    "bench": "math-subset-unspecified",
    "score": 0.954,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'Qwen 3 32B' = 95.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-32B (thinking-model table, Olmo report Table 14).",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "qwen3-32b",
    "bench": "omega",
    "score": 0.477,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'Qwen 3 32B' = 47.7; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-32B (thinking-model table, Olmo report Table 14).",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "math-subset-unspecified",
    "score": 0.967,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'Qwen 3 VL 32B Thinking' = 96.7; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-VL-32B-Thinking.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "qwen3-vl-32b-thinking",
    "bench": "omega",
    "score": 0.508,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'Qwen 3 VL 32B Thinking' = 50.8; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen3-VL-32B-Thinking.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "math-subset-unspecified",
    "score": 0.802,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'Qwen 2.5 32B' = 80.2; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen2.5-32B-Instruct.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "qwen2-5-32b",
    "bench": "omega",
    "score": 0.192,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'Qwen 2.5 32B' = 19.2; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: Qwen/Qwen2.5-32B-Instruct.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "gemma-3-27b",
    "bench": "math-subset-unspecified",
    "score": 0.874,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'Gemma 3 27B Instruct' = 87.4; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: google/gemma-3-27b-it.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "gemma-3-27b",
    "bench": "omega",
    "score": 0.24,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'Gemma 3 27B Instruct' = 24.0; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: google/gemma-3-27b-it.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "gemma-2-27b",
    "bench": "math-subset-unspecified",
    "score": 0.515,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'Gemma 2 27B Instruct' = 51.5; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: google/gemma-2-27b-it.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "gemma-2-27b",
    "bench": "omega",
    "score": 0.091,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'Gemma 2 27B Instruct' = 9.1; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: google/gemma-2-27b-it.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "math-subset-unspecified",
    "score": 0.926,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'MATH', column 'DeepSeek-R1-Distill-Qwen-32B' = 92.6; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "deepseek-r1-distill-qwen-32b",
    "bench": "omega",
    "score": 0.389,
    "source": "https://huggingface.co/allenai/Olmo-3.1-32B-Think/blob/832c3f543499af8fe68b88359501de9cb7840544/README.md",
    "by": "independent",
    "date": "2026-01-05",
    "note": "Olmo-3.1-32B-Think model card @832c3f5 (Apache-2.0), evaluation table, row 'OMEGA', column 'DeepSeek-R1-Distill-Qwen-32B' = 38.9; re-read 2026-10-11. Provenance: Ai2's Olmo 3 technical report (arXiv 2512.13961v2, CC BY 4.0) Sec. 4.1.1: 'We establish a standard evaluation configuration between all baseline models, including thinking and instruct models ... 32K max context length, a sampling temperature of 0.6 and top-p of 0.95', and 'We evaluate all models using our evaluation framework' (Table 15). Checkpoint: deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.",
    "source_id": "card2-allenai-olmo-3-1-32b-think"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "bbh",
    "score": 0.512,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BigBench Hard (0-shot, CoT)', column 'Mistral-3B' = 51.2; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "mmlu",
    "score": 0.608,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU (5-shot)', column 'Mistral-3B' = 60.8; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "mmlu-pro",
    "score": 0.353,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MMLU-Pro (0-shot, CoT)', column 'Mistral-3B' = 35.3; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "arc-challenge",
    "score": 0.803,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'ARC Challenge (10-shot)', column 'Mistral-3B' = 80.3; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "boolq",
    "score": 0.794,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'BoolQ (2-shot)', column 'Mistral-3B' = 79.4; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "gpqa-unspecified",
    "score": 0.244,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GPQA (0-shot, CoT)', column 'Mistral-3B' = 24.4; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "hellaswag",
    "score": 0.746,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'HellaSwag (5-shot)', column 'Mistral-3B' = 74.6; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "openbookqa",
    "score": 0.798,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'OpenBookQA (10-shot)', column 'Mistral-3B' = 79.8; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "piqa",
    "score": 0.732,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'PIQA (5-shot)', column 'Mistral-3B' = 73.2; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "social-iqa",
    "score": 0.739,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Social IQA (5-shot)', column 'Mistral-3B' = 73.9; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "truthfulqa-mc2",
    "score": 0.629,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'TruthfulQA (MC2) (10-shot)', column 'Mistral-3B' = 62.9; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "winogrande",
    "score": 0.598,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Winogrande (5-shot)', column 'Mistral-3B' = 59.8; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "multilingual-mmlu",
    "score": 0.464,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'Multilingual MMLU (5-shot)', column 'Mistral-3B' = 46.4; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "mgsm",
    "score": 0.446,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MGSM (0-shot, CoT)', column 'Mistral-3B' = 44.6; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "gsm8k",
    "score": 0.801,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'GSM8K (8-shot, CoT)', column 'Mistral-3B' = 80.1; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ministral-3b-2410",
    "bench": "math",
    "score": 0.418,
    "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Phi-4-mini-instruct model card (MIT, pinned cfbefac), '### Model Quality' table, row 'MATH (0-shot, CoT)', column 'Mistral-3B' = 41.8; re-read 2026-10-11. Microsoft's own run on its internal benchmark platform (same setup quote as the card's other comparator rows already in data). Identity: the card (created 2025-02-19) labels its other Mistral column 'Mistral-8B-2410' (catalogue ministral-8b-2410); before December 2025 Mistral's only 3B model was Ministral 3B 24.10, so 'Mistral-3B' = ministral-3b-2410 (API-only model; Microsoft ran it through Mistral's API).",
    "source_id": "card-microsoft-phi-4-mini-instruct"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.539,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SWE-Bench Pro', column 'Ring-2.6-1T (xhigh)' = 53.9 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "swe-bench-multilingual",
    "score": 0.567,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SWE-Bench Multilingual', column 'Ring-2.6-1T (xhigh)' = 56.7 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "mcp-atlas-v1-500",
    "score": 0.612,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'MCP-Atlas', column 'Ring-2.6-1T (xhigh)' = 61.2 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "skillsbench-87",
    "score": 0.119,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'Ring-2.6-1T (xhigh)' = 11.9 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "bfcl-v4",
    "score": 0.648,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'BFCL-v4', column 'Ring-2.6-1T (xhigh)' = 64.8 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "widesearch-corrected",
    "score": 0.622,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'Ring-2.6-1T (xhigh)' = 62.2 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "ifbench-unspecified",
    "score": 0.446,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'IFBench', column 'Ring-2.6-1T (xhigh)' = 44.6 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "sysbench-unspecified",
    "score": 0.865,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SysBench', column 'Ring-2.6-1T (xhigh)' = 86.5 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "aime-2026",
    "score": 0.958,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'AIME26', column 'Ring-2.6-1T (xhigh)' = 95.8 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "hmmt-feb-2026",
    "score": 0.935,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'HMMT-Feb26', column 'Ring-2.6-1T (xhigh)' = 93.5 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "imo-answerbench",
    "score": 0.861,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'IMO-AnswerBench', column 'Ring-2.6-1T (xhigh)' = 86.1 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "hle",
    "score": 0.183,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'HLE', column 'Ring-2.6-1T (xhigh)' = 18.3 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "livecodebench-202408-202505-version-unspecified",
    "score": 0.87,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'LiveCodeBench (2408-2505)', column 'Ring-2.6-1T (xhigh)' = 87.0 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "ring-2-6-1t",
    "bench": "multi-if-unspecified",
    "score": 0.893,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'Multi-IF', column 'Ring-2.6-1T (xhigh)' = 89.3 (printed digits in the table image, read 2026-10-11). Same-lab sibling column (inclusionAI / Ant Group's own Ring model) in Ant's own table, same per-benchmark setups as the existing Ling-3.0-flash row. Reasoning effort xhigh (column label).",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "minimax-m2-7",
    "bench": "widesearch-corrected",
    "score": 0.752,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'MiniMax-M2.7' = 75.2; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: catalogue minimax-m2-7.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "minimax-m2-7",
    "bench": "skillsbench-87",
    "score": 0.284,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'MiniMax-M2.7' = 28.4; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: catalogue minimax-m2-7.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "step-3-7-flash",
    "bench": "widesearch-corrected",
    "score": 0.568,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'Step-3.7-Flash (high)' = 56.8; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: catalogue step-3-7-flash, reasoning high.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "step-3-7-flash",
    "bench": "skillsbench-87",
    "score": 0.249,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'Step-3.7-Flash (high)' = 24.9; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: catalogue step-3-7-flash, reasoning high.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "widesearch-corrected",
    "score": 0.744,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'Deepseek-V4-Flash-Preview (max)' = 74.4; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: the April 2026 DeepSeek-V4-Flash preview (catalogue deepseek-v4-flash-preview), reasoning max.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "deepseek-v4-flash-preview",
    "bench": "skillsbench-87",
    "score": 0.535,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'Deepseek-V4-Flash-Preview (max)' = 53.5; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: the April 2026 DeepSeek-V4-Flash preview (catalogue deepseek-v4-flash-preview), reasoning max.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "widesearch-corrected",
    "score": 0.195,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'Nemotron-3-Super-120B-A12B' = 19.5; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: catalogue nemotron-3-super-120b-a12b.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "nemotron-3-super-120b-a12b",
    "bench": "skillsbench-87",
    "score": 0.203,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'Nemotron-3-Super-120B-A12B' = 20.3; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: catalogue nemotron-3-super-120b-a12b.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "widesearch-corrected",
    "score": 0.702,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'GPT-5.4-mini (high)' = 70.2; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: catalogue gpt-5-4-mini, reasoning high.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "gpt-5-4-mini",
    "bench": "skillsbench-87",
    "score": 0.448,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'GPT-5.4-mini (high)' = 44.8; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: catalogue gpt-5-4-mini, reasoning high.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "widesearch-corrected",
    "score": 0.795,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'WideSearch', column 'Claude-Sonnet-4.6 (max)' = 79.5; read 2026-10-11. Ant's own run: card notes 'Search-agent: For all search-agent tasks, evaluations are performed using an internal harness' and 'WideSearch: Evaluated using the official prompt and the official judge model GPT-4.1 on the corrected version of the dataset' (Ant's corrected dataset, so no other publisher has these numbers). Checkpoint: catalogue claude-sonnet-4-6, effort max.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "skillsbench-87",
    "score": 0.544,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'SkillsBench', column 'Claude-Sonnet-4.6 (max)' = 54.4; read 2026-10-11. Ant's own run: card note 'SkillsBench: Evaluated via kilo-code on 87 tasks (excluding external API-dependent tasks), averaged over 3 runs' (Ant's own 87-task subset, so no other publisher has these numbers). Checkpoint: catalogue claude-sonnet-4-6, effort max.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "claude-sonnet-4-6",
    "bench": "browsecomp-ling-single-context",
    "score": 0.74,
    "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md",
    "by": "independent",
    "date": "2026-09-25",
    "note": "Ling-3.0-flash model card @ef06d91 (MIT), evaluation table image linked from the card (https://intranetproxy.alipay.com/skylark/lark/0/2026/png/23157180/1785843565924-387dbbd7-90f8-4241-a58d-664cc5e6b486.png, sha256 a7dc4f02…99817, the same file the existing Ling-3.0-flash rows were transcribed from), row 'BrowseComp', column 'Claude-Sonnet-4.6 (max)' = '74.0 (w/ ctx)' (with 82.1 (MA) below it); read 2026-10-11. Same single-agent context-resume setting as the existing Ling '72.2 (w/ ctx)' row ('BrowseComp (Single-Agent): Evaluated using a resume strategy for context management ... at a 64K-token threshold'); Ant's internal search harness ('For all search-agent tasks, evaluations are performed using an internal harness'). Other columns' single BrowseComp numbers do not say which mode and are not used.",
    "source_id": "card-final-ling-3-0-flash"
  },
  {
    "model": "deepseek-v3-2-speciale",
    "bench": "aime-2025",
    "score": 0.96,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "by": "lab",
    "date": "2025-12-01",
    "note": "DeepSeek-V3.2 model card @a7e62ac (MIT), chart assets/benchmark.png (sha256 08a78f9d…6ffd9, same file as the existing DeepSeek-V3.2 rows), group 'AIME 2025 (Pass@1)', bar 'DeepSeek-V3.2-Speciale' with printed data label 96.0; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab variant in DeepSeek's own chart (the card: 'our high-compute variant, DeepSeek-V3.2-Speciale'). Codeforces rating not used; other labs' bars not used. Bench id follows the existing DeepSeek-V3.2 row of this chart.",
    "source_id": "card2-deepseek-ai-deepseek-v3-2"
  },
  {
    "model": "deepseek-v3-2-speciale",
    "bench": "hmmt-2025-edition-unspecified",
    "score": 0.992,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "by": "lab",
    "date": "2025-12-01",
    "note": "DeepSeek-V3.2 model card @a7e62ac (MIT), chart assets/benchmark.png (sha256 08a78f9d…6ffd9, same file as the existing DeepSeek-V3.2 rows), group 'HMMT 2025 (Pass@1)', bar 'DeepSeek-V3.2-Speciale' with printed data label 99.2; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab variant in DeepSeek's own chart (the card: 'our high-compute variant, DeepSeek-V3.2-Speciale'). Codeforces rating not used; other labs' bars not used. Bench id follows the existing DeepSeek-V3.2 row of this chart.",
    "source_id": "card2-deepseek-ai-deepseek-v3-2"
  },
  {
    "model": "deepseek-v3-2-speciale",
    "bench": "hle",
    "score": 0.306,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md",
    "by": "lab",
    "date": "2025-12-01",
    "note": "DeepSeek-V3.2 model card @a7e62ac (MIT), chart assets/benchmark.png (sha256 08a78f9d…6ffd9, same file as the existing DeepSeek-V3.2 rows), group 'HLE (Pass@1)', bar 'DeepSeek-V3.2-Speciale' with printed data label 30.6; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab variant in DeepSeek's own chart (the card: 'our high-compute variant, DeepSeek-V3.2-Speciale'). Codeforces rating not used; other labs' bars not used. Bench id follows the existing DeepSeek-V3.2 row of this chart.",
    "source_id": "card2-deepseek-ai-deepseek-v3-2"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "tau3-bench",
    "score": 0.645,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'τ³-bench', column 'MiMo-V2-Pro (1.02T / 42B)' = 64.5 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "claw-eval-pass3",
    "score": 0.578,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'ClawEval (pass^3)', column 'MiMo-V2-Pro (1.02T / 42B)' = 57.8 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "hle-with-tools",
    "score": 0.4,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'Humanity's Last Exam (with tools, main figure)', column 'MiMo-V2-Pro (1.02T / 42B)' = 40.0 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "hle",
    "score": 0.28,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'Humanity's Last Exam ('w.o. tools')', column 'MiMo-V2-Pro (1.02T / 42B)' = 28.0 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "swe-bench-pro-split-unspecified",
    "score": 0.55,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'SWE-bench Pro', column 'MiMo-V2-Pro (1.02T / 42B)' = 55.0 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "swe-bench-verified",
    "score": 0.78,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'SWE-bench Verified', column 'MiMo-V2-Pro (1.02T / 42B)' = 78.0 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "mimo-v2-pro",
    "bench": "terminal-bench-2",
    "score": 0.571,
    "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/21d1ecfecd7bd70f31be25ca49d7edd21f003659/README.md",
    "by": "lab",
    "date": "2026-07-09",
    "note": "Xiaomi MiMo-V2.5-Pro model card @21d1ecf, 'Benchmark Results' table image assets/benchmark.jpg (sha256 5b4e4a2c…1a6c6, the same file the existing MiMo-V2.5-Pro rows were transcribed from), row 'Terminal-Bench 2.0', column 'MiMo-V2-Pro (1.02T / 42B)' = 57.1 (printed digits; read 2026-10-11). Same-lab sibling column in Xiaomi's own table; settings not stated, as for the own-model rows. Other labs' columns not used.",
    "source_id": "card2-xiaomimimo-mimo-v2-5-pro"
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2024",
    "score": 0.798,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'AIME 2024 (Pass@1)', column 'DeepSeek R1' = 79.8; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2024",
    "score": 0.914,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'AIME 2024 (Pass@1)', column 'DeepSeek R1 0528' = 91.4; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1",
    "bench": "aime-2025",
    "score": 0.7,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'AIME 2025 (Pass@1)', column 'DeepSeek R1' = 70.0; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "aime-2025",
    "score": 0.875,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'AIME 2025 (Pass@1)', column 'DeepSeek R1 0528' = 87.5; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1",
    "bench": "gpqa-diamond",
    "score": 0.715,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'GPQA-Diamond (Pass@1)', column 'DeepSeek R1' = 71.5; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "gpqa-diamond",
    "score": 0.81,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'GPQA-Diamond (Pass@1)', column 'DeepSeek R1 0528' = 81.0; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.635,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'LiveCodeBench (2408-2505) (Pass@1)', column 'DeepSeek R1' = 63.5; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count. Bench id follows this card's own 'LiveCodeBench (2408-2505)' row (livecodebench-v6-202408-202505); the R1-0528 card files the same label as livecodebench-202408-202505-version-unspecified.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "deepseek-r1-0528",
    "bench": "livecodebench-v6-202408-202505",
    "score": 0.733,
    "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "DeepSeek-R1-0528-Qwen3-8B model card @6e8885a (MIT), first evaluation table (the DeepSeek R1 vs R1-0528 table this card reprints), row 'LiveCodeBench (2408-2505) (Pass@1)', column 'DeepSeek R1 0528' = 73.3; re-read 2026-10-11. Setup: 'For all our models, the maximum generation length is set to 64K tokens ... temperature of 0.6, a top-p value of 0.95, and generate 16 responses per query to estimate pass@1.' Same-lab DeepSeek models. The same figures are also recorded from the DeepSeek-R1-0528 card; kept here too (owner decision 2026-10-11 to keep correlated rows) so this card's own-model runs count. Bench id follows this card's own 'LiveCodeBench (2408-2505)' row (livecodebench-v6-202408-202505); the R1-0528 card files the same label as livecodebench-202408-202505-version-unspecified.",
    "source_id": "card-deepseek-ai-deepseek-r1-0528-qwen3-8b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "swe-bench-pro-qwen-refined",
    "score": 0.535,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'SWE-bench Pro', column 'Qwen3.6-27B' = 53.5; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Card note: 'Problematic tasks were corrected, and all baseline models were re-evaluated on the refined benchmark' (Claude Code harness, temp 1.0, top_p 0.95, 256K context). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "nl2repo",
    "score": 0.362,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'NL2Repo-Bench', column 'Qwen3.6-27B' = 36.2; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Card note: 'Evaluated with the Claude Code harness' with repository-access commands disabled. Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "gpqa-diamond",
    "score": 0.878,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'GPQA Diamond', column 'Qwen3.6-27B' = 87.8; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "hle",
    "score": 0.24,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'HLE', column 'Qwen3.6-27B' = 24.0; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Card note: 'HLE: Judged by GPT-4o.' Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "androidworld",
    "score": 0.703,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'AndroidWorld', column 'Qwen3.6-27B' = 70.3; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "livecodebench-v6-window-unspecified",
    "score": 0.839,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'LiveCodeBench v6', column 'Qwen3.6-27B' = 83.9; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). No window printed (same id as the card's other LCB v6 cells). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "charxiv-rq",
    "score": 0.784,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'CharXiv (RQ)', column 'Qwen3.6-27B' = 78.4; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). The 'Without CI' (no code interpreter) value; the only one printed for Qwen3.6-27B. Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "realworldqa",
    "score": 0.841,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'RealWorldQA', column 'Qwen3.6-27B' = 84.1; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "qwen3-6-27b",
    "bench": "erqa",
    "score": 0.625,
    "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
    "by": "lab",
    "date": "2026-09-25",
    "note": "Qwen3.8-27B model card @1d4bf0f (Apache-2.0), benchmark table, row 'ERQA', column 'Qwen3.6-27B' = 62.5; re-read 2026-10-11. Same-lab column in Qwen's own table (thinking mode by default). Earlier imports skipped this carried-over cell (it may equal the Qwen3.6-27B card's own figure); added now (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "card-qwen--qwen3.8-27b"
  },
  {
    "model": "gemini-3-pro-preview",
    "bench": "mcp-atlas",
    "score": 0.666,
    "source": "https://arxiv.org/html/2602.15763v2",
    "by": "independent",
    "date": "2026-02-24",
    "note": "GLM-5 technical report (Z.ai, arXiv 2602.15763v2, CC BY 4.0), Table 7, row 'MCP-Atlas (Public Set)', column 'Gemini 3 Pro' = 66.6; re-read 2026-10-11. Z.ai's own run: Sec. 6.1.3 'For fair comparison, we re-evaluate all models on the 500-task public set and extend the timeout from 4 minutes to 10 minutes per task'; Appendix B: 'All models are evaluated in think mode ... We use Gemini 3 Pro as the judge model.' Identity: before this February 2026 report Google served Gemini 3 Pro only as gemini-3-pro-preview (the catalogue's only Gemini 3 Pro entry), the same mapping as the other comparator rows in data.",
    "source_id": "report-2602.15763v2"
  },
  {
    "model": "glm-5-2",
    "bench": "terminal-bench-2.1",
    "score": 0.81,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "by": "lab",
    "date": "2026-09-07",
    "note": "Z.ai's own chart embedded by the pinned GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png at zai-org/GLM-5 commit f6adb53, sha256 ea9de778…b372a, same file as the existing rows), group 'Terminal Bench 2.1', bar 'GLM-5.2' (green) with printed data label 81.0; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab sibling bar, same settings as the GLM-5.3-Flash bar (card footnote: Claude Code 2.1.207, temperature 1.0, top_p 1). Other labs' bars not used.",
    "source_id": "card2-zai-org-glm-5-3-flash"
  },
  {
    "model": "glm-5-2",
    "bench": "hle-with-tools",
    "score": 0.547,
    "source": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md",
    "by": "lab",
    "date": "2026-09-07",
    "note": "Z.ai's own chart embedded by the pinned GLM-5.3-Flash README ('LLM Performance Evaluation'; resources/bench_53.png at zai-org/GLM-5 commit f6adb53, sha256 ea9de778…b372a, same file as the existing rows), group 'HLE w/ Tools', bar 'GLM-5.2' (green) with printed data label 54.7; read 2026-10-11 (printed label, owner policy 2026-10-09). Same-lab sibling bar, same settings as the GLM-5.3-Flash bar (card footnote: Claude Code 2.1.207, temperature 1.0, top_p 1). Other labs' bars not used.",
    "source_id": "card2-zai-org-glm-5-3-flash"
  },
  {
    "model": "whisper-large-v3",
    "bench": "ami-wer",
    "score": 0.1595,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'AMI', column 'large-v3' = 15.95 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "ami-wer",
    "score": 0.1613,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'AMI', column 'large-v3-turbo' = 16.13 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "gigaspeech-wer",
    "score": 0.1002,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Gigaspeech', column 'large-v3' = 10.02 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "gigaspeech-wer",
    "score": 0.1014,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Gigaspeech', column 'large-v3-turbo' = 10.14 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-clean-wer",
    "score": 0.0201,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'LS Clean', column 'large-v3' = 2.01 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "librispeech-test-clean-wer",
    "score": 0.021,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'LS Clean', column 'large-v3-turbo' = 2.1 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "librispeech-test-other-wer",
    "score": 0.0391,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'LS Other', column 'large-v3' = 3.91 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "librispeech-test-other-wer",
    "score": 0.0424,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'LS Other', column 'large-v3-turbo' = 4.24 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "tedlium-wer",
    "score": 0.0386,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Tedlium', column 'large-v3' = 3.86 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "tedlium-wer",
    "score": 0.0357,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Tedlium', column 'large-v3-turbo' = 3.57 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "earnings22-wer",
    "score": 0.1129,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Earnings22', column 'large-v3' = 11.29 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "earnings22-wer",
    "score": 0.1163,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'Earnings22', column 'large-v3-turbo' = 11.63 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "spgispeech-wer",
    "score": 0.0294,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'SPGISpeech', column 'large-v3' = 2.94 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "spgispeech-wer",
    "score": 0.0297,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Short-Form Evaluation' table, row 'SPGISpeech', column 'large-v3-turbo' = 2.97 (WER %); re-read 2026-10-11. Hugging Face's own measurement: the card authors evaluated 'as done in the Open ASR Leaderboard' and this cell equals Hugging Face's Open ASR Leaderboard CSV (en_shortform.csv @16743c6, Apache-2.0, already in the ledger as open-asr), i.e. HF's run of OpenAI's openai/whisper-large-v3-turbo. Same table and normalisation as the existing distil rows of this card. Also feeds open-asr-wer's average (correlated; owner decision 2026-10-11 to keep).",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "tedlium-longform-wer",
    "score": 0.0307,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Long-Form Evaluation' table, row 'tedlium-long-form', column 'large-v3-turbo' = 3.07 (WER %); re-read 2026-10-11. Card: 'We've evaluated the model on 1 in-distribution (ID) test sets and 4 out-of-distribution (OOD) test sets for long-form transcription, using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; the turbo column differs from the Open ASR long-form CSV (e.g. earnings22 13.1 there), so it is the card authors' own sequential-decoding run of openai/whisper-large-v3-turbo alongside the distil models.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "meanwhile-longform-wer",
    "score": 0.0503,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Long-Form Evaluation' table, row 'meanwhile', column 'large-v3-turbo' = 5.03 (WER %); re-read 2026-10-11. Card: 'We've evaluated the model on 1 in-distribution (ID) test sets and 4 out-of-distribution (OOD) test sets for long-form transcription, using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; the turbo column differs from the Open ASR long-form CSV (e.g. earnings22 13.1 there), so it is the card authors' own sequential-decoding run of openai/whisper-large-v3-turbo alongside the distil models.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "earnings21-longform-wer",
    "score": 0.0984,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Long-Form Evaluation' table, row 'earnings21', column 'large-v3-turbo' = 9.84 (WER %); re-read 2026-10-11. Card: 'We've evaluated the model on 1 in-distribution (ID) test sets and 4 out-of-distribution (OOD) test sets for long-form transcription, using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; the turbo column differs from the Open ASR long-form CSV (e.g. earnings22 13.1 there), so it is the card authors' own sequential-decoding run of openai/whisper-large-v3-turbo alongside the distil models.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "earnings22-longform-wer",
    "score": 0.1332,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Long-Form Evaluation' table, row 'earnings22', column 'large-v3-turbo' = 13.32 (WER %); re-read 2026-10-11. Card: 'We've evaluated the model on 1 in-distribution (ID) test sets and 4 out-of-distribution (OOD) test sets for long-form transcription, using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; the turbo column differs from the Open ASR long-form CSV (e.g. earnings22 13.1 there), so it is the card authors' own sequential-decoding run of openai/whisper-large-v3-turbo alongside the distil models.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3-turbo",
    "bench": "rev16-longform-wer",
    "score": 0.1282,
    "source": "https://huggingface.co/distil-whisper/distil-large-v3.5/blob/728a7691f3ff1d3d971528d3203a6e9559165d41/README.md",
    "by": "independent",
    "date": "2026-04-13",
    "note": "distil-whisper/distil-large-v3.5 model card @728a769 (MIT), 'Long-Form Evaluation' table, row 'rev16', column 'large-v3-turbo' = 12.82 (WER %); re-read 2026-10-11. Card: 'We've evaluated the model on 1 in-distribution (ID) test sets and 4 out-of-distribution (OOD) test sets for long-form transcription, using the sequential decoding algorithm (condition_on_prev_tokens=False, return_timestamps=True)'; the turbo column differs from the Open ASR long-form CSV (e.g. earnings22 13.1 there), so it is the card authors' own sequential-decoding run of openai/whisper-large-v3-turbo alongside the distil models.",
    "source_id": "card2-distil-whisper-distil-large-v3-5"
  },
  {
    "model": "whisper-large-v3",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.125876,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T11",
    "by": "independent",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 11 'Per-language WER (%) results on the FLEURS benchmark', row 'whisper-large-v3': derived unweighted mean of the 25 printed per-language cells over the 25 FLEURS languages (bg ... uk) = 12.5876% (cells: 12.86, 11.33, 12.57, 4.30, 27.03, 4.25, 3.12, 19.12, 7.70, 6.31, 11.07, 14.11, 2.31, 22.34, 18.29, 68.89, 5.57, 4.74, 3.65, 8.24, 4.17, 8.40, 19.93, 7.88, 6.51); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). NVIDIA's own runs: Sec. 6.1 'For comparative analysis, we consider several strong baselines: Whisper-large-v3 (1.55B), seamless-m4t-v2-large (2.3B), and seamless-m4t-medium (1.2B) ...' and 'we report normalized WER (%). We use the Hugging Face Open ASR Leaderboard normalizers ... and the newly added multilingual normalizer for non-English test sets', so no other publisher has these baseline numbers. Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.119696,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T11",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 11 'Per-language WER (%) results on the FLEURS benchmark', row 'Parakeet-TDT-0.6B-v3': derived unweighted mean of the 25 printed per-language cells over the 25 FLEURS languages (bg ... uk) = 11.9696% (cells: 12.64, 11.01, 18.41, 5.04, 20.70, 4.85, 3.45, 17.73, 13.21, 5.15, 12.46, 15.72, 3.00, 20.35, 22.84, 20.46, 7.48, 7.31, 4.76, 12.44, 5.51, 8.82, 24.03, 15.08, 6.79); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "canary-1b-v2",
    "bench": "fleurs-25eu-avg-wer",
    "score": 0.083964,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T11",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 11 'Per-language WER (%) results on the FLEURS benchmark', row 'Canary-1B-v2': derived unweighted mean of the 25 printed per-language cells over the 25 FLEURS languages (bg ... uk) = 8.3964% (cells: 9.25, 7.86, 11.25, 4.40, 9.21, 4.50, 2.90, 12.55, 8.59, 5.02, 8.29, 12.90, 3.07, 12.36, 9.66, 18.31, 6.12, 6.64, 4.39, 6.61, 6.90, 5.74, 13.32, 9.57, 10.50); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "mls-6lang-avg-wer",
    "score": 0.080367,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T12",
    "by": "independent",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 12 'Per-language WER (%) results on the MLS benchmark', row 'whisper-large-v3': derived unweighted mean of the 6 printed per-language cells over the 6 MLS languages (es, fr, it, nl, pl, pt) = 8.0367% (cells: 4.89, 7.15, 9.26, 12.08, 8.88, 5.96); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). NVIDIA's own runs: Sec. 6.1 'For comparative analysis, we consider several strong baselines: Whisper-large-v3 (1.55B), seamless-m4t-v2-large (2.3B), and seamless-m4t-medium (1.2B) ...' and 'we report normalized WER (%). We use the Hugging Face Open ASR Leaderboard normalizers ... and the newly added multilingual normalizer for non-English test sets', so no other publisher has these baseline numbers. Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "seamless-m4t-v2-large",
    "bench": "mls-6lang-avg-wer",
    "score": 0.053583,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T12",
    "by": "independent",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 12 'Per-language WER (%) results on the MLS benchmark', row 'seamless-m4t-v2-large': derived unweighted mean of the 6 printed per-language cells over the 6 MLS languages (es, fr, it, nl, pl, pt) = 5.3583% (cells: 3.08, 3.73, 6.83, 6.91, 5.74, 5.86); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). NVIDIA's own runs: Sec. 6.1 'For comparative analysis, we consider several strong baselines: Whisper-large-v3 (1.55B), seamless-m4t-v2-large (2.3B), and seamless-m4t-medium (1.2B) ...' and 'we report normalized WER (%). We use the Hugging Face Open ASR Leaderboard normalizers ... and the newly added multilingual normalizer for non-English test sets', so no other publisher has these baseline numbers. Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "mls-6lang-avg-wer",
    "score": 0.078333,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T12",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 12 'Per-language WER (%) results on the MLS benchmark', row 'Parakeet-TDT-0.6B-v3': derived unweighted mean of the 6 printed per-language cells over the 6 MLS languages (es, fr, it, nl, pl, pt) = 7.8333% (cells: 4.39, 4.97, 10.08, 12.78, 7.28, 7.50); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "canary-1b-v2",
    "bench": "mls-6lang-avg-wer",
    "score": 0.072733,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T12",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 12 'Per-language WER (%) results on the MLS benchmark', row 'Canary-1B-v2': derived unweighted mean of the 6 printed per-language cells over the 6 MLS languages (es, fr, it, nl, pl, pt) = 7.2733% (cells: 2.94, 3.36, 9.16, 11.27, 8.77, 8.14); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "whisper-large-v3",
    "bench": "covost2-13lang-avg-wer",
    "score": 0.102746,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T13",
    "by": "independent",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 13 'Per-language WER (%) results on the CoVoST2 benchmark', row 'whisper-large-v3': derived unweighted mean of the 13 printed per-language cells over the 13 CoVoST2 languages (de, en, es, et, fr, it, lv, nl, pt, ru, sl, sv, uk) = 10.2746% (cells: 5.94, 8.52, 4.32, 28.54, 11.02, 5.45, 18.84, 5.97, 3.72, 4.04, 16.63, 8.06, 12.52); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). NVIDIA's own runs: Sec. 6.1 'For comparative analysis, we consider several strong baselines: Whisper-large-v3 (1.55B), seamless-m4t-v2-large (2.3B), and seamless-m4t-medium (1.2B) ...' and 'we report normalized WER (%). We use the Hugging Face Open ASR Leaderboard normalizers ... and the newly added multilingual normalizer for non-English test sets', so no other publisher has these baseline numbers. Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "parakeet-tdt-0-6b-v3",
    "bench": "covost2-13lang-avg-wer",
    "score": 0.119777,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T13",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 13 'Per-language WER (%) results on the CoVoST2 benchmark', row 'Parakeet-TDT-0.6B-v3': derived unweighted mean of the 13 printed per-language cells over the 13 CoVoST2 languages (de, en, es, et, fr, it, lv, nl, pt, ru, sl, sv, uk) = 11.9777% (cells: 4.84, 6.80, 3.41, 22.04, 6.05, 3.69, 38.36, 6.50, 3.96, 3.00, 31.80, 20.16, 5.10); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "canary-1b-v2",
    "bench": "covost2-13lang-avg-wer",
    "score": 0.088508,
    "source": "https://arxiv.org/html/2509.14128v2#A2.T13",
    "by": "lab",
    "date": "2025-09-25",
    "note": "Canary-1B-v2 & Parakeet-TDT-0.6B-v3 report (NVIDIA, arXiv 2509.14128v2, CC BY-SA 4.0), Appendix Table 13 'Per-language WER (%) results on the CoVoST2 benchmark', row 'Canary-1B-v2': derived unweighted mean of the 13 printed per-language cells over the 13 CoVoST2 languages (de, en, es, et, fr, it, lv, nl, pt, ru, sl, sv, uk) = 8.8508% (cells: 5.53, 6.85, 3.81, 18.28, 6.30, 4.80, 11.49, 6.93, 6.87, 5.14, 7.59, 13.32, 18.15); read 2026-10-11. Same language set and averaging as the benchmark definition (for Parakeet/Canary the mean reproduces the model cards' printed averages). Own model (by: lab). Share-alike: also in this file's extracts.",
    "source_id": "paper-2509.14128v2-canary-parakeet"
  },
  {
    "model": "grok-voice-transcribe-1-0",
    "bench": "raven-de-read4-wer",
    "score": 0.083625,
    "source": "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/BENCHMARKS.md",
    "by": "independent",
    "date": "2026-09-18",
    "note": "raven.eval BENCHMARKS.md @f9f621e (MIT), section 'WER — German ASR (public datasets)', rows 'xai/grok-voice-transcribe-1.0': avemio-german-mixed-test 7.21, fleurs 6.75, mls-de 8.22, voxpopuli-de 11.27 (n=100 each); re-read 2026-10-11. Derived unweighted mean of the four committed expected.json wer_pct values (7.2067, 6.7545, 8.2212, 11.2676 -> 8.3625%), the same derivation as the existing grok-voice-transcribe-2 row (Swiss spc-test / fhnw sets excluded as in the benchmark definition). The raven.eval author's own run: 'Its predecessor Grok Voice Transcribe 1.0 ran on the same samples with the same settings (500 of 500, 1.50 h, USD 0.15)'. Checkpoint: xAI slug grok-voice-transcribe-1.0 (catalogue grok-voice-transcribe-1-0, now retired).",
    "source_id": "speech-stt-raven-mit"
  },
  {
    "model": "seedream-4-0",
    "bench": "chineseword-longcat-overall",
    "score": 0.585,
    "source": "https://arxiv.org/html/2512.07584v1#S5.T7",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan, arXiv 2512.07584v1, CC BY 4.0), Table 7 'Quantitative evaluation results of ChineseWord', row 'Seedream 4.0 (Seedream et al., 2025)', column 'Overall' = 58.5; read 2026-10-11. The authors built this test: 'we constructed a comprehensive benchmark comprising 8,105 prompts based on the General Standard Chinese Characters Table ... aligning with the protocol of Qwen-Image ... We employ PPOCRv5 for objective accuracy quantification', so every cell is Meituan's own run (a test the document introduces). Checkpoint: ByteDance Seedream 4.0 (catalogue seedream-4-0).",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "qwen-image",
    "bench": "chineseword-longcat-overall",
    "score": 0.566,
    "source": "https://arxiv.org/html/2512.07584v1#S5.T7",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan, arXiv 2512.07584v1, CC BY 4.0), Table 7 'Quantitative evaluation results of ChineseWord', row 'Qwen-Image (Wu et al., 2025a)', column 'Overall' = 56.6; read 2026-10-11. The authors built this test: 'we constructed a comprehensive benchmark comprising 8,105 prompts based on the General Standard Chinese Characters Table ... aligning with the protocol of Qwen-Image ... We employ PPOCRv5 for objective accuracy quantification', so every cell is Meituan's own run (a test the document introduces). Checkpoint: Qwen/Qwen-Image (catalogue qwen-image).",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "hunyuanimage-3-0",
    "bench": "chineseword-longcat-overall",
    "score": 0.493,
    "source": "https://arxiv.org/html/2512.07584v1#S5.T7",
    "by": "independent",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan, arXiv 2512.07584v1, CC BY 4.0), Table 7 'Quantitative evaluation results of ChineseWord', row 'HunyuanImage-3.0 (Cao et al., 2025)', column 'Overall' = 49.3; read 2026-10-11. The authors built this test: 'we constructed a comprehensive benchmark comprising 8,105 prompts based on the General Standard Chinese Characters Table ... aligning with the protocol of Qwen-Image ... We employ PPOCRv5 for objective accuracy quantification', so every cell is Meituan's own run (a test the document introduces). Checkpoint: tencent/HunyuanImage-3.0 (the September 2025 release; the Instruct variant came after this December 2025 report).",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "longcat-image",
    "bench": "chineseword-longcat-overall",
    "score": 0.907,
    "source": "https://arxiv.org/html/2512.07584v1#S5.T7",
    "by": "lab",
    "date": "2025-12-08",
    "note": "LongCat-Image report (Meituan, arXiv 2512.07584v1, CC BY 4.0), Table 7 'Quantitative evaluation results of ChineseWord', row 'LongCat-Image', column 'Overall' = 90.7; read 2026-10-11. The authors built this test: 'we constructed a comprehensive benchmark comprising 8,105 prompts based on the General Standard Chinese Characters Table ... aligning with the protocol of Qwen-Image ... We employ PPOCRv5 for objective accuracy quantification', so every cell is Meituan's own run (a test the document introduces). Checkpoint: own model.",
    "source_id": "report-2512.07584v1"
  },
  {
    "model": "xai-grok-tts",
    "bench": "vapi-humanness-export-20260610",
    "score": 1306.53,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'xai-xai-tts': elo 1306.53, wins 71, losses 24, ties 3, voteCount 98; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (xai-grok-tts). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "cartesia-sonic-3-5",
    "bench": "vapi-humanness-export-20260610",
    "score": 1257.23,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'cartesia-sonic-35': elo 1257.23, wins 56, losses 32, ties 9, voteCount 97; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (cartesia-sonic-3-5). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "elevenlabs-v3",
    "bench": "vapi-humanness-export-20260610",
    "score": 1231.83,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'elevenlabs-eleven-v3': elo 1231.83, wins 55, losses 41, ties 5, voteCount 101; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (elevenlabs-v3). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "elevenlabs-flash-v2-5",
    "bench": "vapi-humanness-export-20260610",
    "score": 1225.44,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'elevenlabs-flash-v25': elo 1225.44, wins 52, losses 40, ties 8, voteCount 100; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (elevenlabs-flash-v2-5). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "elevenlabs-flash-v2",
    "bench": "vapi-humanness-export-20260610",
    "score": 1208.12,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'elevenlabs-flash-v2': elo 1208.12, wins 48, losses 43, ties 6, voteCount 97; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (elevenlabs-flash-v2). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "elevenlabs-turbo-v2",
    "bench": "vapi-humanness-export-20260610",
    "score": 1201.25,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'elevenlabs-turbo-v2': elo 1201.25, wins 46, losses 40, ties 11, voteCount 97; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (elevenlabs-turbo-v2). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "inworld-tts-1-5-max",
    "bench": "vapi-humanness-export-20260610",
    "score": 1197.31,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'inworld-tts-15-max': elo 1197.31, wins 2, losses 3, ties 0, voteCount 5; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (inworld-tts-1-5-max). Only 5 votes in this export. The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "inworld-realtime-tts-2",
    "bench": "vapi-humanness-export-20260610",
    "score": 1189.42,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'inworld-tts-2': elo 1189.42, wins 1, losses 4, ties 0, voteCount 5; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (inworld-realtime-tts-2). Only 5 votes in this export. The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "cartesia-sonic-2",
    "bench": "vapi-humanness-export-20260610",
    "score": 1139.56,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'cartesia-sonic-2': elo 1139.56, wins 36, losses 59, ties 2, voteCount 97; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (cartesia-sonic-2). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "gradium-tts",
    "bench": "vapi-humanness-export-20260610",
    "score": 1098.63,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'gradium-gradium-tts': elo 1098.63, wins 20, losses 69, ties 6, voteCount 95; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (gradium-tts). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "cartesia-sonic-3",
    "bench": "vapi-humanness-export-20260610",
    "score": 1096.65,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'cartesia-sonic-3': elo 1096.65, wins 27, losses 61, ties 8, voteCount 96; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (cartesia-sonic-3). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "cartesia-sonic",
    "bench": "vapi-humanness-export-20260610",
    "score": 1027.55,
    "source": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json",
    "by": "independent",
    "date": "2026-06-10",
    "note": "Vapi Humanness Index seed-standings.json @e2b8f86 (numerical vote data CC BY 4.0), exportedAt '2026-06-10 production leaderboard export from the original prototype deployment', entry 'cartesia-sonic': elo 1027.55, wins 13, losses 82, ties 4, voteCount 99; read 2026-10-11. Registry id mapped exactly as the existing September 2026 rows of this source family map it (cartesia-sonic). The same June votes are anchor games inside Vapi's September BT fit, so this row is correlated with the model's vapi-humanness-bt-20260925 row (owner decision 2026-10-11 to keep correlated rows).",
    "source_id": "speech-vapi-humanness-20260610"
  },
  {
    "model": "mochi-1",
    "bench": "wan-bench-2-1-weighted",
    "score": 0.639,
    "source": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B/blob/a064a6c71f5be440641209c07bf2a5ce7a2ff5e4/README.md",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Wan2.1-T2V-14B model card @a064a6c (Apache-2.0), 'Comparisons to SOTA' table image assets/vben_vs_sota.png (sha256 9a0e86ca…da44e, the same file the existing Wan-14B row was read from), row 'Weighted Score', column 'Mochi' = 0.639 (printed digits; read 2026-10-11). Wan team's own run: 'Using our carefully designed set of 1,035 internal prompts, we tested across 14 major dimensions and 26 sub-dimensions. We then compute the total score by performing a weighted calculation ...' (an internal test the card introduces). Checkpoint: Genmo Mochi 1 (preview), the only Mochi release before this February 2025 card (catalogue mochi-1). Anonymised CNTopA/B/C columns not used.",
    "source_id": "card2-wan-ai-wan2-1-t2v-14b"
  },
  {
    "model": "sora",
    "bench": "wan-bench-2-1-weighted",
    "score": 0.7,
    "source": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B/blob/a064a6c71f5be440641209c07bf2a5ce7a2ff5e4/README.md",
    "by": "independent",
    "date": "2025-03-12",
    "note": "Wan2.1-T2V-14B model card @a064a6c (Apache-2.0), 'Comparisons to SOTA' table image assets/vben_vs_sota.png (sha256 9a0e86ca…da44e, the same file the existing Wan-14B row was read from), row 'Weighted Score', column 'Sora' = 0.7 (printed digits; read 2026-10-11). Wan team's own run: 'Using our carefully designed set of 1,035 internal prompts, we tested across 14 major dimensions and 26 sub-dimensions. We then compute the total score by performing a weighted calculation ...' (an internal test the card introduces). Checkpoint: OpenAI's original Sora product (December 2024; Sora 2 came October 2025), catalogue sora. Anonymised CNTopA/B/C columns not used.",
    "source_id": "card2-wan-ai-wan2-1-t2v-14b"
  },
  {
    "model": "kandinsky-2-0",
    "bench": "coco30k-fid-kandinsky21",
    "score": 20,
    "source": "https://huggingface.co/kandinsky-community/kandinsky-2-1/blob/04b366659663bf60deacb96fd3e38f1cb39a3be4/README.md",
    "by": "lab",
    "date": "2026-10-06",
    "note": "Kandinsky 2.1 card @ 04b3666, '### Evaluation' table 'FID metric values for generative models on COCO_30k', row 'Kandinsky 2.0 (2022)', column 'FID (30k)' = 20.00 (raw distance, lower is better). Same table as the existing Kandinsky 2.1 row (8.21). Kandinsky 2.0 is the same Sber AI / AI Forever team's previous model and the row is not marked as copied, so it is taken as a lab measurement under the brief's same-family rule; caveat: the card's setup sentence ('We quantitatively measure the performance of Kandinsky 2.1 on the COCO_30k dataset, in zero-shot mode') names only 2.1, so the 2.0 figure's generation settings are not restated. Checkpoint: ai-forever/Kandinsky_2.0 @ 04e8fc7ea945965edfcb33705ac272d14b9d0f6c (proposed comparison-only kandinsky-2-0). Date is review date, as for the existing row. Re-read 2026-10-11 at the pinned card (row 'Kandinsky 2.0 (2022)' = 20.00).",
    "source_id": "kandinsky21-card-original-fid"
  },
  {
    "model": "grok-2-0813",
    "bench": "livebench-20240831",
    "score": 0.4655,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'grok-2'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; dates from the six question datasets at their pinned 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Recomputed 2026-10-11 with LiveBench's show_livebench_result.py aggregation (task means -> category means -> mean of six categories) = 46.55; category means: math 42.7, coding 36.2, reasoning 38.0, language 34.8, instruction_following 74.8, data_analysis 52.7. The same script reproduces the existing gemini-1-5-flash-002 (49.57) and o1-mini (59.09) rows. Answers generated and scored by the LiveBench team. Identity: judgments timestamped 2024-10-17..22, before Grok-2 1212 existed (December 2024), so 'grok-2' is the August 2024 Grok-2 family (catalogue grok-2-0813, Arena 'grok-2-2024-08-13'). No 2024-07-26 result exists for this model, as the benchmark's one-release-per-model rule requires.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "grok-2-mini-0813",
    "bench": "livebench-20240831",
    "score": 0.4554,
    "source": "https://huggingface.co/datasets/livebench/model_judgment/blob/5896e3b11081702c7f93f4733605fa4f5a072a11/data/leaderboard-00000-of-00001.parquet",
    "by": "independent",
    "date": "2024-10-22",
    "note": "LiveBench model judgments (HF livebench/model_judgment @5896e3b, committed 2024-10-22), model 'grok-2-mini'. Release 2024-08-31 question set (1,000 questions: livebench_release_date in {2024-06-24, 2024-07-26, 2024-08-31} and not removed by 2024-08-31; dates from the six question datasets at their pinned 2024-10-22 revisions): judgments for all 1,000 questions, no duplicates. Recomputed 2026-10-11 with LiveBench's show_livebench_result.py aggregation (task means -> category means -> mean of six categories) = 45.54; category means: math 40.3, coding 37.5, reasoning 42.0, language 39.5, instruction_following 69.7, data_analysis 44.3. The same script reproduces the existing gemini-1-5-flash-002 (49.57) and o1-mini (59.09) rows. Answers generated and scored by the LiveBench team. Identity: judgments timestamped 2024-10-17..22, before Grok-2 1212 existed (December 2024), so 'grok-2-mini' is the August 2024 Grok-2 family (catalogue grok-2-mini-0813, Arena 'grok-2-mini-2024-08-13'). No 2024-07-26 result exists for this model, as the benchmark's one-release-per-model rule requires.",
    "source_id": "livebench-model-judgment-apache"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "bigcodebench-complete",
    "score": 0.582,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 6 'Evaluating LLMs on BigCodeBench-Complete', row 'GPT-4-Turbo' / checkpoint '2024-04-09', column 'Greedy Decoding · Original' = 0.582 (calibrated 0.582); re-read 2026-10-11. Original greedy pass@1, as for every other row of this source. Authors' own runs (Sec. 4). Identity: both results tables print checkpoint 2024-04-09 next to the number and cite OpenAI's 2024 GPT-4 Turbo page; the artifacts Table 4 instead lists 'GPT-4-Turbo | gpt-4-0125-preview', but the same table maps GPT-4o to gpt-4-turbo-2024-04-09, an evident row shift, so the results tables' checkpoint is the better mapping (catalogue gpt-4-turbo-2024-04-09).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "gpt-4-turbo-2024-04-09",
    "bench": "bigcodebench-instruct",
    "score": 0.48,
    "source": "https://arxiv.org/html/2406.15877v4",
    "by": "independent",
    "date": "2025-04-01",
    "note": "BigCodeBench paper (Zhuo et al., arXiv 2406.15877v4, CC BY 4.0), Table 7 'Evaluating instruction-tuned LLMs on BigCodeBench-Instruct', row 'GPT-4-Turbo' / checkpoint '2024-04-09', column 'Greedy Decoding · Original' = 0.480 (calibrated 0.482); re-read 2026-10-11. Original greedy pass@1, as for every other row of this source. Authors' own runs (Sec. 4). Identity: both results tables print checkpoint 2024-04-09 next to the number and cite OpenAI's 2024 GPT-4 Turbo page; the artifacts Table 4 instead lists 'GPT-4-Turbo | gpt-4-0125-preview', but the same table maps GPT-4o to gpt-4-turbo-2024-04-09, an evident row shift, so the results tables' checkpoint is the better mapping (catalogue gpt-4-turbo-2024-04-09).",
    "source_id": "paper-2406.15877v4-bigcodebench"
  },
  {
    "model": "cohere-transcribe-arabic-07-2026",
    "bench": "aura-ar-en-22set-avg-wer",
    "score": 0.2087,
    "source": "https://huggingface.co/spaces/QCRI/AURA-Arabic-English-ASR-leaderboard/blob/208ff13439f377131207a2b32bf73364ba0e2d0c/leaderboard.tsv",
    "by": "independent",
    "date": "2026-10-06",
    "note": "QCRI 'ASR Leaderboard — Arabic + English short-form' (Space QCRI/AURA-Arabic-English-ASR-leaderboard, cc-by-4.0), leaderboard.tsv @208ff13 (committed 2026-10-06), row 'cohere-transcribe-arabic-07-2026', column 'Average WER' = 20.87 (Average CER 10.45); read 2026-10-11. QCRI's own scoring: the Space states 'All reported WER/CER scores use the same bilingual Arabic–English text normalizer before scoring' (its own normalizer, so it scored every system's transcripts itself). Average over the 22 sets in the file (arzen, casablanca, common_voice_ar, commonvoice_ar_aura, dacs, escwa_cs, everyayah, fleurs_ar/cs/en, l2_arctic, librispeech clean/other, masc clean/noisy, MGB2/3/5, mixat_tri, qasr_cs, rdi, sada). Checkpoint: catalogue cohere-transcribe-arabic-07-2026.",
    "source_id": "speech-stt-qcri-aura-ccby"
  },
  {
    "model": "qwen3-asr-1-7b",
    "bench": "aura-ar-en-22set-avg-wer",
    "score": 0.2935,
    "source": "https://huggingface.co/spaces/QCRI/AURA-Arabic-English-ASR-leaderboard/blob/208ff13439f377131207a2b32bf73364ba0e2d0c/leaderboard.tsv",
    "by": "independent",
    "date": "2026-10-06",
    "note": "QCRI 'ASR Leaderboard — Arabic + English short-form' (Space QCRI/AURA-Arabic-English-ASR-leaderboard, cc-by-4.0), leaderboard.tsv @208ff13 (committed 2026-10-06), row 'qwen3_asr_1_7b', column 'Average WER' = 29.35 (Average CER 13.56); read 2026-10-11. QCRI's own scoring: the Space states 'All reported WER/CER scores use the same bilingual Arabic–English text normalizer before scoring' (its own normalizer, so it scored every system's transcripts itself). Average over the 22 sets in the file (arzen, casablanca, common_voice_ar, commonvoice_ar_aura, dacs, escwa_cs, everyayah, fleurs_ar/cs/en, l2_arctic, librispeech clean/other, masc clean/noisy, MGB2/3/5, mixat_tri, qasr_cs, rdi, sada). Checkpoint: catalogue qwen3-asr-1-7b.",
    "source_id": "speech-stt-qcri-aura-ccby"
  },
  {
    "model": "qwen3-asr-0-6b",
    "bench": "aura-ar-en-22set-avg-wer",
    "score": 0.366,
    "source": "https://huggingface.co/spaces/QCRI/AURA-Arabic-English-ASR-leaderboard/blob/208ff13439f377131207a2b32bf73364ba0e2d0c/leaderboard.tsv",
    "by": "independent",
    "date": "2026-10-06",
    "note": "QCRI 'ASR Leaderboard — Arabic + English short-form' (Space QCRI/AURA-Arabic-English-ASR-leaderboard, cc-by-4.0), leaderboard.tsv @208ff13 (committed 2026-10-06), row 'qwen3_asr_0_6b', column 'Average WER' = 36.6 (Average CER 16.46); read 2026-10-11. QCRI's own scoring: the Space states 'All reported WER/CER scores use the same bilingual Arabic–English text normalizer before scoring' (its own normalizer, so it scored every system's transcripts itself). Average over the 22 sets in the file (arzen, casablanca, common_voice_ar, commonvoice_ar_aura, dacs, escwa_cs, everyayah, fleurs_ar/cs/en, l2_arctic, librispeech clean/other, masc clean/noisy, MGB2/3/5, mixat_tri, qasr_cs, rdi, sada). Checkpoint: catalogue qwen3-asr-0-6b.",
    "source_id": "speech-stt-qcri-aura-ccby"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.751,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Pro (05-06 preview)', column 'VoxCeleb2 (EM)' = 0.751; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.581,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2.5-Omni (7B)', column 'VoxCeleb2 (EM)' = 0.581; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.529,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash', column 'VoxCeleb2 (EM)' = 0.529; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.53,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash (Experimental)', column 'VoxCeleb2 (EM)' = 0.530; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.759,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Flash (05-20 preview)', column 'VoxCeleb2 (EM)' = 0.759; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.575,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-12-17)', column 'VoxCeleb2 (EM)' = 0.575; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.57,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-10-01)', column 'VoxCeleb2 (EM)' = 0.570; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.585,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (002)', column 'VoxCeleb2 (EM)' = 0.585; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.24,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2-Audio Instruct (7B)', column 'VoxCeleb2 (EM)' = 0.240; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.527,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash Lite', column 'VoxCeleb2 (EM)' = 0.527; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.542,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (002)', column 'VoxCeleb2 (EM)' = 0.542; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.509,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'VoxCeleb2 (EM)' = 0.509; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.524,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (001)', column 'VoxCeleb2 (EM)' = 0.524; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-voxceleb2-em",
    "score": 0.522,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (001)', column 'VoxCeleb2 (EM)' = 0.522; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.86,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Pro (05-06 preview)', column 'VocalSound (PEM)' = 0.860; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.904,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2.5-Omni (7B)', column 'VocalSound (PEM)' = 0.904; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.719,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash', column 'VocalSound (PEM)' = 0.719; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.718,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash (Experimental)', column 'VocalSound (PEM)' = 0.718; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.626,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Flash (05-20 preview)', column 'VocalSound (PEM)' = 0.626; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.837,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-12-17)', column 'VocalSound (PEM)' = 0.837; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.833,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-10-01)', column 'VocalSound (PEM)' = 0.833; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.528,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (002)', column 'VocalSound (PEM)' = 0.528; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.799,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2-Audio Instruct (7B)', column 'VocalSound (PEM)' = 0.799; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.506,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash Lite', column 'VocalSound (PEM)' = 0.506; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.418,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (002)', column 'VocalSound (PEM)' = 0.418; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.794,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'VocalSound (PEM)' = 0.794; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.492,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (001)', column 'VocalSound (PEM)' = 0.492; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-vocalsound-pem",
    "score": 0.463,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (001)', column 'VocalSound (PEM)' = 0.463; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-librispeech-wer",
    "score": 0.039,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Pro (05-06 preview)', column 'LibriSpeech (WER)' = 0.039; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-librispeech-wer",
    "score": 0.103,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2.5-Omni (7B)', column 'LibriSpeech (WER)' = 0.103; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-librispeech-wer",
    "score": 0.043,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash', column 'LibriSpeech (WER)' = 0.043; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-librispeech-wer",
    "score": 0.044,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash (Experimental)', column 'LibriSpeech (WER)' = 0.044; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-librispeech-wer",
    "score": 0.077,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.5 Flash (05-20 preview)', column 'LibriSpeech (WER)' = 0.077; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-librispeech-wer",
    "score": 0.095,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-12-17)', column 'LibriSpeech (WER)' = 0.095; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-librispeech-wer",
    "score": 0.113,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o Audio (Preview 2024-10-01)', column 'LibriSpeech (WER)' = 0.113; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-librispeech-wer",
    "score": 0.052,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (002)', column 'LibriSpeech (WER)' = 0.052; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-librispeech-wer",
    "score": 0.113,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Qwen2-Audio Instruct (7B)', column 'LibriSpeech (WER)' = 0.113; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-librispeech-wer",
    "score": 0.049,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 2.0 Flash Lite', column 'LibriSpeech (WER)' = 0.049; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-librispeech-wer",
    "score": 0.062,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (002)', column 'LibriSpeech (WER)' = 0.062; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-librispeech-wer",
    "score": 0.163,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'LibriSpeech (WER)' = 0.163; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-librispeech-wer",
    "score": 0.071,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Pro (001)', column 'LibriSpeech (WER)' = 0.071; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-librispeech-wer",
    "score": 0.342,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T8",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A8 (audio perception), row 'Gemini 1.5 Flash (001)', column 'LibriSpeech (WER)' = 0.342; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.724,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Qwen2-Audio Instruct (7B)', column 'Air-Bench Foundation (EM)' = 0.724; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.683,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 2.5 Pro (05-06 preview)', column 'Air-Bench Foundation (EM)' = 0.683; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.697,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 2.0 Flash', column 'Air-Bench Foundation (EM)' = 0.697; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.579,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 2.5 Flash (05-20 preview)', column 'Air-Bench Foundation (EM)' = 0.579; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.698,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 2.0 Flash (Experimental)', column 'Air-Bench Foundation (EM)' = 0.698; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.743,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Qwen2.5-Omni (7B)', column 'Air-Bench Foundation (EM)' = 0.743; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.56,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'GPT-4o Audio (Preview 2024-12-17)', column 'Air-Bench Foundation (EM)' = 0.560; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.641,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 2.0 Flash Lite', column 'Air-Bench Foundation (EM)' = 0.641; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.527,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'GPT-4o Audio (Preview 2024-10-01)', column 'Air-Bench Foundation (EM)' = 0.527; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.554,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 1.5 Pro (002)', column 'Air-Bench Foundation (EM)' = 0.554; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.541,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'Air-Bench Foundation (EM)' = 0.541; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.508,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 1.5 Flash (002)', column 'Air-Bench Foundation (EM)' = 0.508; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.483,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 1.5 Flash (001)', column 'Air-Bench Foundation (EM)' = 0.483; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-airbench-foundation-music-em",
    "score": 0.511,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T9",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A9 (knowledge), row 'Gemini 1.5 Pro (001)', column 'Air-Bench Foundation (EM)' = 0.511; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-corebench-pem",
    "score": 0.813,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 2.5 Pro (05-06 preview)', column 'COREBench (PEM)' = 0.813; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-corebench-pem",
    "score": 0.756,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 2.0 Flash', column 'COREBench (PEM)' = 0.756; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-corebench-pem",
    "score": 0.799,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 1.5 Pro (002)', column 'COREBench (PEM)' = 0.799; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-corebench-pem",
    "score": 0.754,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 2.0 Flash (Experimental)', column 'COREBench (PEM)' = 0.754; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-corebench-pem",
    "score": 0.776,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 1.5 Flash (002)', column 'COREBench (PEM)' = 0.776; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-corebench-pem",
    "score": 0.644,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 2.5 Flash (05-20 preview)', column 'COREBench (PEM)' = 0.644; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-corebench-pem",
    "score": 0.737,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 2.0 Flash Lite', column 'COREBench (PEM)' = 0.737; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-corebench-pem",
    "score": 0.722,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 1.5 Flash (001)', column 'COREBench (PEM)' = 0.722; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-corebench-pem",
    "score": 0.659,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Gemini 1.5 Pro (001)', column 'COREBench (PEM)' = 0.659; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-corebench-pem",
    "score": 0.233,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Qwen2-Audio Instruct (7B)', column 'COREBench (PEM)' = 0.233; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-corebench-pem",
    "score": 0.359,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'GPT-4o Audio (Preview 2024-12-17)', column 'COREBench (PEM)' = 0.359; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-corebench-pem",
    "score": 0.56,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'Qwen2.5-Omni (7B)', column 'COREBench (PEM)' = 0.560; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-corebench-pem",
    "score": 0.514,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'COREBench (PEM)' = 0.514; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-corebench-pem",
    "score": 0.342,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T10",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A10 (reasoning), row 'GPT-4o Audio (Preview 2024-10-01)', column 'COREBench (PEM)' = 0.342; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.473,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.5 Pro (05-06 preview)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.473; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.497,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o Audio (Preview 2024-12-17)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.497; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.491,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Qwen2.5-Omni (7B)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.491; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.516,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Pro (002)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.516; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.368,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash Lite', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.368; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.443,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash (Experimental)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.443; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.456,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o Audio (Preview 2024-10-01)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.456; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.423,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.423; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.334,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.334; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.469,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Pro (001)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.469; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.471,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Flash (001)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.471; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.34,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.5 Flash (05-20 preview)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.340; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.425,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Flash (002)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.425; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-meld-audio-pem",
    "score": 0.26,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Qwen2-Audio Instruct (7B)', column 'Multimodal EmotionLines Dataset (MELD) Audio (PEM)' = 0.260; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-mustard-em",
    "score": 0.655,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.5 Pro (05-06 preview)', column 'MUStARD (EM)' = 0.655; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-mustard-em",
    "score": 0.583,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o Audio (Preview 2024-12-17)', column 'MUStARD (EM)' = 0.583; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-mustard-em",
    "score": 0.588,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Qwen2.5-Omni (7B)', column 'MUStARD (EM)' = 0.588; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-mustard-em",
    "score": 0.577,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Pro (002)', column 'MUStARD (EM)' = 0.577; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-mustard-em",
    "score": 0.661,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash Lite', column 'MUStARD (EM)' = 0.661; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-mustard-em",
    "score": 0.604,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash (Experimental)', column 'MUStARD (EM)' = 0.604; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-mustard-em",
    "score": 0.593,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o Audio (Preview 2024-10-01)', column 'MUStARD (EM)' = 0.593; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-mustard-em",
    "score": 0.604,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.0 Flash', column 'MUStARD (EM)' = 0.604; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-mustard-em",
    "score": 0.623,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'MUStARD (EM)' = 0.623; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-mustard-em",
    "score": 0.564,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Pro (001)', column 'MUStARD (EM)' = 0.564; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-mustard-em",
    "score": 0.555,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Flash (001)', column 'MUStARD (EM)' = 0.555; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-mustard-em",
    "score": 0.574,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 2.5 Flash (05-20 preview)', column 'MUStARD (EM)' = 0.574; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-mustard-em",
    "score": 0.558,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Gemini 1.5 Flash (002)', column 'MUStARD (EM)' = 0.558; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-mustard-em",
    "score": 0.209,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T11",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A11 (emotion detection), row 'Qwen2-Audio Instruct (7B)', column 'MUStARD (EM)' = 0.209; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.342,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Pro (002)', column 'FLEURS (WER)' = 0.342; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.211,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.5 Pro (05-06 preview)', column 'FLEURS (WER)' = 0.211; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.648,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash', column 'FLEURS (WER)' = 0.648; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.443,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash Lite', column 'FLEURS (WER)' = 0.443; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.646,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash (Experimental)', column 'FLEURS (WER)' = 0.646; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.456,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o Audio (Preview 2024-12-17)', column 'FLEURS (WER)' = 0.456; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.463,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Pro (001)', column 'FLEURS (WER)' = 0.463; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.461,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Flash (002)', column 'FLEURS (WER)' = 0.461; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.545,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'FLEURS (WER)' = 0.545; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.723,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Flash (001)', column 'FLEURS (WER)' = 0.723; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 2.732,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.5 Flash (05-20 preview)', column 'FLEURS (WER)' = 2.732; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 0.771,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o Audio (Preview 2024-10-01)', column 'FLEURS (WER)' = 0.771; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 2.24,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Qwen2-Audio Instruct (7B)', column 'FLEURS (WER)' = 2.240; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-fleurs-7lang-wer",
    "score": 1.932,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Qwen2.5-Omni (7B)', column 'FLEURS (WER)' = 1.932; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.054,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Pro (002)', column 'Multilingual Librispeech (WER)' = 0.054; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.198,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.5 Pro (05-06 preview)', column 'Multilingual Librispeech (WER)' = 0.198; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.06,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash', column 'Multilingual Librispeech (WER)' = 0.060; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.067,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash Lite', column 'Multilingual Librispeech (WER)' = 0.067; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.06,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.0 Flash (Experimental)', column 'Multilingual Librispeech (WER)' = 0.060; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.073,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o Audio (Preview 2024-12-17)', column 'Multilingual Librispeech (WER)' = 0.073; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.073,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Pro (001)', column 'Multilingual Librispeech (WER)' = 0.073; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.071,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Flash (002)', column 'Multilingual Librispeech (WER)' = 0.071; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.123,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'Multilingual Librispeech (WER)' = 0.123; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.088,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 1.5 Flash (001)', column 'Multilingual Librispeech (WER)' = 0.088; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.603,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Gemini 2.5 Flash (05-20 preview)', column 'Multilingual Librispeech (WER)' = 0.603; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.162,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'GPT-4o Audio (Preview 2024-10-01)', column 'Multilingual Librispeech (WER)' = 0.162; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.337,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Qwen2-Audio Instruct (7B)', column 'Multilingual Librispeech (WER)' = 0.337; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-mls-7lang-wer",
    "score": 0.416,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T15",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A15 (multilinguality), row 'Qwen2.5-Omni (7B)', column 'Multilingual Librispeech (WER)' = 0.416; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-pro-preview-05-06",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.039,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 2.5 Pro (05-06 preview)', column 'Robust Speech Bench (WER)' = 0.039; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-pro-preview-05-06.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-lite-001",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.049,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 2.0 Flash Lite', column 'Robust Speech Bench (WER)' = 0.049; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-lite-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-5-flash-preview-05-20",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.077,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 2.5 Flash (05-20 preview)', column 'Robust Speech Bench (WER)' = 0.077; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.5-flash-preview-05-20.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-5-omni-7b",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.103,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Qwen2.5-Omni (7B)', column 'Robust Speech Bench (WER)' = 0.103; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2.5-omni-7b.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-exp",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.171,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 2.0 Flash (Experimental)', column 'Robust Speech Bench (WER)' = 0.171; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-exp.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-2-0-flash-001",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.178,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 2.0 Flash', column 'Robust Speech Bench (WER)' = 0.178; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-2.0-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-002",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.207,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 1.5 Pro (002)', column 'Robust Speech Bench (WER)' = 0.207; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-pro-001",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.213,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 1.5 Pro (001)', column 'Robust Speech Bench (WER)' = 0.213; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-pro-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-002",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.214,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 1.5 Flash (002)', column 'Robust Speech Bench (WER)' = 0.214; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-002.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "qwen2-audio-7b-instruct",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.399,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Qwen2-Audio Instruct (7B)', column 'Robust Speech Bench (WER)' = 0.399; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: qwen2-audio-7b-instruct.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-12-17",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.451,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'GPT-4o Audio (Preview 2024-12-17)', column 'Robust Speech Bench (WER)' = 0.451; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-mini-audio-preview-2024-12-17",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.471,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'GPT-4o mini Audio (Preview 2024-12-17)', column 'Robust Speech Bench (WER)' = 0.471; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-mini-audio-preview-2024-12-17.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gemini-1-5-flash-001",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.498,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'Gemini 1.5 Flash (001)', column 'Robust Speech Bench (WER)' = 0.498; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gemini-1.5-flash-001.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-4o-audio-preview-2024-10-01",
    "bench": "ahelm-srb-librispeech-noise-wer",
    "score": 0.822,
    "source": "https://arxiv.org/html/2508.21376v2#A8.T19",
    "by": "independent",
    "date": "2025-09-02",
    "note": "AHELM: A Holistic Evaluation of Audio-Language Models (Stanford CRFM, arXiv 2508.21376v2, CC BY 4.0), Table A19 (robustness), row 'GPT-4o Audio (Preview 2024-10-01)', column 'Robust Speech Bench (WER)' = 0.822; read 2026-10-11. The authors' own HELM runs of every model ('we standardize the inference parameters by setting the model temperature to 0 and the maximum number of output tokens to 200'; prompts and raw outputs released). Identifier per Table 3: gpt-4o-audio-preview-2024-10-01.",
    "source_id": "paper-2508.21376v2-ahelm"
  },
  {
    "model": "gpt-j-6b",
    "bench": "truthfulqa-mc2",
    "score": 0.3596,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-j-6b/results_2023-12-08%2020%3A56%3A47.673590.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-j-6b/results_2023-12-08 20:56:47.673590.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.359571; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-j-6b,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-j-6b (catalogue gpt-j-6b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-j-6b",
    "bench": "halueval-qa-acc",
    "score": 0.5136,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-j-6b/results_2024-03-03%2014%3A34%3A24.699661.json",
    "by": "independent",
    "date": "2024-03-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-j-6b/results_2024-03-03 14:34:24.699661.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.513600; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-j-6b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-j-6b (catalogue gpt-j-6b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-j-6b",
    "bench": "triviaqa",
    "score": 0.3932,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-j-6b/results_2023-12-18%2022%3A11%3A14.689150.json",
    "by": "independent",
    "date": "2023-12-18",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-j-6b/results_2023-12-18 22:11:14.689150.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.393168; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-j-6b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-j-6b (catalogue gpt-j-6b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-j-6b",
    "bench": "natural-questions",
    "score": 0.0917,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-j-6b/results_2024-01-03%2010%3A52%3A57.129715.json",
    "by": "independent",
    "date": "2024-01-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-j-6b/results_2024-01-03 10:52:57.129715.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.091690; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-j-6b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-j-6b (catalogue gpt-j-6b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "truthfulqa-mc2",
    "score": 0.3961,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-1.3B/results_2023-12-08%2021%3A38%3A29.151646.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-1.3B/results_2023-12-08 21:38:29.151646.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.396138; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-1.3B,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-1.3B (catalogue gpt-neo-1-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "halueval-qa-acc",
    "score": 0.4625,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-1.3B/results_2023-12-13%2013%3A02%3A51.572572.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-1.3B/results_2023-12-13 13:02:51.572572.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.462500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-1.3B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-1.3B (catalogue gpt-neo-1-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "triviaqa",
    "score": 0.1416,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-1.3B/results_2023-12-23%2012%3A22%3A01.380579.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-1.3B/results_2023-12-23 12:22:01.380579.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.141551; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-1.3B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-1.3B (catalogue gpt-neo-1-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-1-3b",
    "bench": "natural-questions",
    "score": 0.0399,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-1.3B/results_2024-02-11%2016%3A59%3A09.001434.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-1.3B/results_2024-02-11 16:59:09.001434.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.039889; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-1.3B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-1.3B (catalogue gpt-neo-1-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-125m",
    "bench": "truthfulqa-mc2",
    "score": 0.4558,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-125m/results_2023-12-08%2019%3A24%3A13.797490.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-125m/results_2023-12-08 19:24:13.797490.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.455794; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-125m,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-125m (catalogue gpt-neo-125m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-125m",
    "bench": "halueval-qa-acc",
    "score": 0.4653,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-125m/results_2023-12-15%2008%3A24%3A04.254953.json",
    "by": "independent",
    "date": "2023-12-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-125m/results_2023-12-15 08:24:04.254953.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.465300; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-125m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-125m (catalogue gpt-neo-125m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-125m",
    "bench": "natural-questions",
    "score": 0.0053,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-125m/results_2024-02-12%2009%3A09%3A48.570004.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-125m/results_2024-02-12 09:09:48.570004.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.005263; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-125m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-125m (catalogue gpt-neo-125m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-125m",
    "bench": "triviaqa",
    "score": 0.0155,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-125m/results_2024-03-01%2012%3A27%3A02.075398.json",
    "by": "independent",
    "date": "2024-03-01",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-125m/results_2024-03-01 12:27:02.075398.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.015548; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-125m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-125m (catalogue gpt-neo-125m, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.3986,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-2.7B/results_2024-03-24%2012%3A16%3A36.444857.json",
    "by": "independent",
    "date": "2024-03-24",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-2.7B/results_2024-03-24 12:16:36.444857.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.398626; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-2.7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-2.7B (catalogue gpt-neo-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "halueval-qa-acc",
    "score": 0.4318,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-2.7B/results_2024-02-15%2010%3A17%3A29.004994.json",
    "by": "independent",
    "date": "2024-02-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-2.7B/results_2024-02-15 10:17:29.004994.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.431800; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-2.7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-2.7B (catalogue gpt-neo-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "triviaqa",
    "score": 0.2011,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-2.7B/results_2024-02-26%2021%3A09%3A06.217241.json",
    "by": "independent",
    "date": "2024-02-26",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-2.7B/results_2024-02-26 21:09:06.217241.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.201070; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-2.7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-2.7B (catalogue gpt-neo-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gpt-neo-2-7b",
    "bench": "natural-questions",
    "score": 0.0571,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/EleutherAI/gpt-neo-2.7B/results_2024-03-06%2021%3A19%3A57.134182.json",
    "by": "independent",
    "date": "2024-03-06",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'EleutherAI/gpt-neo-2.7B/results_2024-03-06 21:19:57.134182.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.057064; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=EleutherAI/gpt-neo-2.7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo EleutherAI/gpt-neo-2.7B (catalogue gpt-neo-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "truthfulqa-mc2",
    "score": 0.5602,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-alpha/results_2023-12-08%2013%3A03%3A17.650169.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-alpha/results_2023-12-08 13:03:17.650169.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.560223; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-alpha,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-alpha (catalogue zephyr-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "halueval-qa-acc",
    "score": 0.6235,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-alpha/results_2023-12-13%2018%3A25%3A28.889934.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-alpha/results_2023-12-13 18:25:28.889934.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.623500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-alpha (catalogue zephyr-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "triviaqa",
    "score": 0.6592,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-alpha/results_2023-12-19%2016%3A57%3A22.897308.json",
    "by": "independent",
    "date": "2023-12-19",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-alpha/results_2023-12-19 16:57:22.897308.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.659162; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-alpha (catalogue zephyr-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-alpha",
    "bench": "natural-questions",
    "score": 0.2623,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-alpha/results_2024-02-12%2016%3A24%3A44.695702.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-alpha/results_2024-02-12 16:24:44.695702.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.262327; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-alpha (catalogue zephyr-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "truthfulqa-mc2",
    "score": 0.5512,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-beta/results_2023-12-08%2022%3A58%3A30.457732.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-beta/results_2023-12-08 22:58:30.457732.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.551195; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-beta,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-beta (catalogue zephyr-7b-beta, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "halueval-qa-acc",
    "score": 0.5185,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-beta/results_2023-12-15%2001%3A27%3A07.699123.json",
    "by": "independent",
    "date": "2023-12-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-beta/results_2023-12-15 01:27:07.699123.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.518500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-beta,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-beta (catalogue zephyr-7b-beta, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "triviaqa",
    "score": 0.6443,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-beta/results_2023-12-29%2001%3A37%3A53.771889.json",
    "by": "independent",
    "date": "2023-12-29",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-beta/results_2023-12-29 01:37:53.771889.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.644338; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-beta,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-beta (catalogue zephyr-7b-beta, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "zephyr-7b-beta",
    "bench": "natural-questions",
    "score": 0.2521,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/HuggingFaceH4/zephyr-7b-beta/results_2024-02-11%2013%3A29%3A45.375857.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'HuggingFaceH4/zephyr-7b-beta/results_2024-02-11 13:29:45.375857.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.252078; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=HuggingFaceH4/zephyr-7b-beta,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo HuggingFaceH4/zephyr-7b-beta (catalogue zephyr-7b-beta, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "nexusraven-v2-13b",
    "bench": "truthfulqa-mc2",
    "score": 0.4439,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/Nexusflow/NexusRaven-V2-13B/results_2024-01-30%2014%3A29%3A26.599300.json",
    "by": "independent",
    "date": "2024-01-30",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'Nexusflow/NexusRaven-V2-13B/results_2024-01-30 14:29:26.599300.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.443925; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=Nexusflow/NexusRaven-V2-13B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo Nexusflow/NexusRaven-V2-13B (catalogue nexusraven-v2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "nexusraven-v2-13b",
    "bench": "halueval-qa-acc",
    "score": 0.2373,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/Nexusflow/NexusRaven-V2-13B/results_2024-02-03%2010%3A10%3A35.690425.json",
    "by": "independent",
    "date": "2024-02-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'Nexusflow/NexusRaven-V2-13B/results_2024-02-03 10:10:35.690425.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.237300; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=Nexusflow/NexusRaven-V2-13B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo Nexusflow/NexusRaven-V2-13B (catalogue nexusraven-v2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "nexusraven-v2-13b",
    "bench": "triviaqa",
    "score": 0.4229,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/Nexusflow/NexusRaven-V2-13B/results_2024-02-08%2018%3A51%3A34.994633.json",
    "by": "independent",
    "date": "2024-02-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'Nexusflow/NexusRaven-V2-13B/results_2024-02-08 18:51:34.994633.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.422871; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=Nexusflow/NexusRaven-V2-13B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo Nexusflow/NexusRaven-V2-13B (catalogue nexusraven-v2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "nexusraven-v2-13b",
    "bench": "natural-questions",
    "score": 0.1421,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/Nexusflow/NexusRaven-V2-13B/results_2024-03-03%2011%3A06%3A20.668175.json",
    "by": "independent",
    "date": "2024-03-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'Nexusflow/NexusRaven-V2-13B/results_2024-03-03 11:06:20.668175.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.142105; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=Nexusflow/NexusRaven-V2-13B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo Nexusflow/NexusRaven-V2-13B (catalogue nexusraven-v2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "truthfulqa-mc2",
    "score": 0.4731,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/berkeley-nest/Starling-LM-7B-alpha/results_2024-02-25%2012%3A45%3A15.485715.json",
    "by": "independent",
    "date": "2024-02-25",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'berkeley-nest/Starling-LM-7B-alpha/results_2024-02-25 12:45:15.485715.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.473102; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=berkeley-nest/Starling-LM-7B-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo berkeley-nest/Starling-LM-7B-alpha (catalogue starling-lm-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "halueval-qa-acc",
    "score": 0.5969,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/berkeley-nest/Starling-LM-7B-alpha/results_2023-12-21%2009%3A43%3A40.146761.json",
    "by": "independent",
    "date": "2023-12-21",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'berkeley-nest/Starling-LM-7B-alpha/results_2023-12-21 09:43:40.146761.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.596900; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=berkeley-nest/Starling-LM-7B-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo berkeley-nest/Starling-LM-7B-alpha (catalogue starling-lm-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "triviaqa",
    "score": 0.6595,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/berkeley-nest/Starling-LM-7B-alpha/results_2023-12-22%2011%3A21%3A54.818457.json",
    "by": "independent",
    "date": "2023-12-22",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'berkeley-nest/Starling-LM-7B-alpha/results_2023-12-22 11:21:54.818457.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.659496; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=berkeley-nest/Starling-LM-7B-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo berkeley-nest/Starling-LM-7B-alpha (catalogue starling-lm-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "starling-lm-7b-alpha",
    "bench": "natural-questions",
    "score": 0.2825,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/berkeley-nest/Starling-LM-7B-alpha/results_2024-02-11%2023%3A09%3A48.144665.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'berkeley-nest/Starling-LM-7B-alpha/results_2024-02-11 23:09:48.144665.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.282548; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=berkeley-nest/Starling-LM-7B-alpha,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo berkeley-nest/Starling-LM-7B-alpha (catalogue starling-lm-7b-alpha, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b1",
    "bench": "truthfulqa-mc2",
    "score": 0.418,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b1/results_2023-12-08%2015%3A05%3A15.296709.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b1/results_2023-12-08 15:05:15.296709.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.417998; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b1,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b1 (catalogue bloom-1b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b1",
    "bench": "halueval-qa-acc",
    "score": 0.4489,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b1/results_2023-12-13%2020%3A48%3A36.485842.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b1/results_2023-12-13 20:48:36.485842.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.448900; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b1 (catalogue bloom-1b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b1",
    "bench": "triviaqa",
    "score": 0.086,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b1/results_2023-12-23%2000%3A02%3A01.385072.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b1/results_2023-12-23 00:02:01.385072.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.085990; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b1 (catalogue bloom-1b1, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b1",
    "bench": "natural-questions",
    "score": 0.0285,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b1/results_2024-04-18%2002%3A15%3A36.261617.json",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b1/results_2024-04-18 02:15:36.261617.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.028532; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b1 (catalogue bloom-1b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b7",
    "bench": "truthfulqa-mc2",
    "score": 0.4132,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b7/results_2024-03-15%2020%3A16%3A48.786685.json",
    "by": "independent",
    "date": "2024-03-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b7/results_2024-03-15 20:16:48.786685.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.413181; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b7,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b7 (catalogue bloom-1b7, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b7",
    "bench": "halueval-qa-acc",
    "score": 0.4953,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b7/results_2023-12-13%2014%3A06%3A29.091254.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b7/results_2023-12-13 14:06:29.091254.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.495300; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b7,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b7 (catalogue bloom-1b7, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b7",
    "bench": "triviaqa",
    "score": 0.1266,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b7/results_2023-12-21%2013%3A02%3A13.597525.json",
    "by": "independent",
    "date": "2023-12-21",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b7/results_2023-12-21 13:02:13.597525.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.126616; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b7,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b7 (catalogue bloom-1b7, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-1b7",
    "bench": "natural-questions",
    "score": 0.0352,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-1b7/results_2024-02-12%2000%3A59%3A11.499700.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-1b7/results_2024-02-12 00:59:11.499700.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.035180; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-1b7,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-1b7 (catalogue bloom-1b7, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-3b",
    "bench": "truthfulqa-mc2",
    "score": 0.4057,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-3b/results_2023-12-08%2018%3A34%3A33.500404.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-3b/results_2023-12-08 18:34:33.500404.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.405722; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-3b,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-3b (catalogue bloom-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-3b",
    "bench": "halueval-qa-acc",
    "score": 0.5039,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-3b/results_2023-12-13%2007%3A14%3A36.270327.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-3b/results_2023-12-13 07:14:36.270327.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.503900; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-3b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-3b (catalogue bloom-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-3b",
    "bench": "triviaqa",
    "score": 0.1788,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-3b/results_2023-12-21%2003%3A43%3A18.750581.json",
    "by": "independent",
    "date": "2023-12-21",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-3b/results_2023-12-21 03:43:18.750581.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.178834; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-3b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-3b (catalogue bloom-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-3b",
    "bench": "natural-questions",
    "score": 0.0479,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-3b/results_2024-05-19%2014%3A27%3A27.344882.json",
    "by": "independent",
    "date": "2024-05-19",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-3b/results_2024-05-19 14:27:27.344882.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.047922; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-3b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-3b (catalogue bloom-3b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-560m",
    "bench": "truthfulqa-mc2",
    "score": 0.4243,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-560m/results_2023-12-08%2022%3A05%3A34.909248.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-560m/results_2023-12-08 22:05:34.909248.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.424315; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-560m,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-560m (catalogue bloom-560m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-560m",
    "bench": "halueval-qa-acc",
    "score": 0.4995,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-560m/results_2023-12-13%2019%3A21%3A11.652926.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-560m/results_2023-12-13 19:21:11.652926.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.499500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-560m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-560m (catalogue bloom-560m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-560m",
    "bench": "natural-questions",
    "score": 0.0161,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-560m/results_2024-02-12%2009%3A30%3A18.175863.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-560m/results_2024-02-12 09:30:18.175863.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.016066; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-560m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-560m (catalogue bloom-560m, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-560m",
    "bench": "triviaqa",
    "score": 0.0571,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-560m/results_2024-02-12%2012%3A17%3A00.645926.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-560m/results_2024-02-12 12:17:00.645926.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.057066; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-560m,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-560m (catalogue bloom-560m, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-7b1",
    "bench": "truthfulqa-mc2",
    "score": 0.3889,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-7b1/results_2023-12-08%2010%3A14%3A09.198300.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-7b1/results_2023-12-08 10:14:09.198300.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.388947; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-7b1,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-7b1 (catalogue bloom-7b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-7b1",
    "bench": "halueval-qa-acc",
    "score": 0.5806,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-7b1/results_2023-12-16%2017%3A25%3A02.092045.json",
    "by": "independent",
    "date": "2023-12-16",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-7b1/results_2023-12-16 17:25:02.092045.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.580600; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-7b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-7b1 (catalogue bloom-7b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-7b1",
    "bench": "triviaqa",
    "score": 0.2634,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-7b1/results_2023-12-23%2002%3A01%3A26.108634.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-7b1/results_2023-12-23 02:01:26.108634.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.263375; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-7b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-7b1 (catalogue bloom-7b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "bloom-7b1",
    "bench": "natural-questions",
    "score": 0.0814,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/bigscience/bloom-7b1/results_2024-02-26%2019%3A43%3A19.008314.json",
    "by": "independent",
    "date": "2024-02-26",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'bigscience/bloom-7b1/results_2024-02-26 19:43:19.008314.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.081440; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=bigscience/bloom-7b1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo bigscience/bloom-7b1 (catalogue bloom-7b1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "opt-13b",
    "bench": "truthfulqa-mc2",
    "score": 0.3401,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/facebook/opt-13b/results_2023-12-22%2023%3A28%3A13.543609.json",
    "by": "independent",
    "date": "2023-12-22",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'facebook/opt-13b/results_2023-12-22 23:28:13.543609.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.340111; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=facebook/opt-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo facebook/opt-13b (catalogue opt-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "opt-13b",
    "bench": "natural-questions",
    "score": 0.1349,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/facebook/opt-13b/results_2023-12-23%2005%3A24%3A22.772239.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'facebook/opt-13b/results_2023-12-23 05:24:22.772239.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.134903; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=facebook/opt-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo facebook/opt-13b (catalogue opt-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "opt-13b",
    "bench": "halueval-qa-acc",
    "score": 0.3532,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/facebook/opt-13b/results_2023-12-28%2004%3A38%3A57.693135.json",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'facebook/opt-13b/results_2023-12-28 04:38:57.693135.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.353200; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=facebook/opt-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo facebook/opt-13b (catalogue opt-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "opt-13b",
    "bench": "triviaqa",
    "score": 0.4101,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/facebook/opt-13b/results_2024-05-19%2018%3A26%3A05.712584.json",
    "by": "independent",
    "date": "2024-05-19",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'facebook/opt-13b/results_2024-05-19 18:26:05.712584.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.410053; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=facebook/opt-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo facebook/opt-13b (catalogue opt-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "triviaqa",
    "score": 0.4777,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/google/gemma-2b/results_2024-03-17%2019%3A52%3A39.397717.json",
    "by": "independent",
    "date": "2024-03-17",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'google/gemma-2b/results_2024-03-17 19:52:39.397717.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.477708; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=google/gemma-2b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "natural-questions",
    "score": 0.1476,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/google/gemma-2b/results_2024-04-04%2021%3A13%3A00.669268.json",
    "by": "independent",
    "date": "2024-04-04",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'google/gemma-2b/results_2024-04-04 21:13:00.669268.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.147645; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=google/gemma-2b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "truthfulqa-mc2",
    "score": 0.3309,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/google/gemma-2b/results_2024-04-05%2004%3A23%3A04.350707.json",
    "by": "independent",
    "date": "2024-04-05",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'google/gemma-2b/results_2024-04-05 04:23:04.350707.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.330891; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=google/gemma-2b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "halueval-qa-acc",
    "score": 0.4861,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/google/gemma-2b/results_2024-04-18%2003%3A19%3A22.548788.json",
    "by": "independent",
    "date": "2024-04-18",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'google/gemma-2b/results_2024-04-18 03:19:22.548788.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.486100; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=google/gemma-2b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "gemma-7b-base",
    "bench": "truthfulqa-mc2",
    "score": 0.4489,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/google/gemma-7b/results_2024-03-15%2022%3A12%3A39.316927.json",
    "by": "independent",
    "date": "2024-03-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'google/gemma-7b/results_2024-03-15 22:12:39.316927.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.448895; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=google/gemma-7b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo google/gemma-7b (catalogue gemma-7b-base, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "triviaqa",
    "score": 0.5636,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/lmsys/vicuna-7b-v1.5/results_2024-02-28%2009%3A28%3A43.055893.json",
    "by": "independent",
    "date": "2024-02-28",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'lmsys/vicuna-7b-v1.5/results_2024-02-28 09:28:43.055893.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.563642; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=lmsys/vicuna-7b-v1.5,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo lmsys/vicuna-7b-v1.5 (catalogue vicuna-7b-v1-5, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "natural-questions",
    "score": 0.2064,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/lmsys/vicuna-7b-v1.5/results_2024-01-31%2009%3A55%3A35.942145.json",
    "by": "independent",
    "date": "2024-01-31",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'lmsys/vicuna-7b-v1.5/results_2024-01-31 09:55:35.942145.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.206371; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=lmsys/vicuna-7b-v1.5,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo lmsys/vicuna-7b-v1.5 (catalogue vicuna-7b-v1-5, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "truthfulqa-mc2",
    "score": 0.5039,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/lmsys/vicuna-7b-v1.5/results_2024-02-01%2011%3A38%3A22.224835.json",
    "by": "independent",
    "date": "2024-02-01",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'lmsys/vicuna-7b-v1.5/results_2024-02-01 11:38:22.224835.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.503944; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=lmsys/vicuna-7b-v1.5,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo lmsys/vicuna-7b-v1.5 (catalogue vicuna-7b-v1-5, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "vicuna-7b-v1-5",
    "bench": "halueval-qa-acc",
    "score": 0.3809,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/lmsys/vicuna-7b-v1.5/results_2024-02-01%2015%3A42%3A45.011389.json",
    "by": "independent",
    "date": "2024-02-01",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'lmsys/vicuna-7b-v1.5/results_2024-02-01 15:42:45.011389.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.380900; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=lmsys/vicuna-7b-v1.5,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo lmsys/vicuna-7b-v1.5 (catalogue vicuna-7b-v1-5, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "truthfulqa-mc2",
    "score": 0.4395,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-chat-hf/results_2023-12-09%2001%3A44%3A28.882454.json",
    "by": "independent",
    "date": "2023-12-09",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-chat-hf/results_2023-12-09 01:44:28.882454.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.439461; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-chat-hf,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-chat-hf (catalogue llama-2-13b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "halueval-qa-acc",
    "score": 0.5728,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-chat-hf/results_2023-12-16%2006%3A20%3A50.987097.json",
    "by": "independent",
    "date": "2023-12-16",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-chat-hf/results_2023-12-16 06:20:50.987097.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.572800; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-chat-hf (catalogue llama-2-13b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "triviaqa",
    "score": 0.0979,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-chat-hf/results_2023-12-21%2012%3A28%3A18.410155.json",
    "by": "independent",
    "date": "2023-12-21",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-chat-hf/results_2023-12-21 12:28:18.410155.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.097860; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-chat-hf (catalogue llama-2-13b-chat, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b-chat",
    "bench": "natural-questions",
    "score": 0.2729,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-chat-hf/results_2024-02-13%2007%3A12%3A03.086723.json",
    "by": "independent",
    "date": "2024-02-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-chat-hf/results_2024-02-13 07:12:03.086723.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.272853; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-chat-hf (catalogue llama-2-13b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b",
    "bench": "truthfulqa-mc2",
    "score": 0.3689,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-hf/results_2023-12-08%2021%3A46%3A18.652384.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-hf/results_2023-12-08 21:46:18.652384.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.368926; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-hf,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-hf (catalogue llama-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b",
    "bench": "halueval-qa-acc",
    "score": 0.7005,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-hf/results_2024-02-13%2003%3A33%3A02.904705.json",
    "by": "independent",
    "date": "2024-02-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-hf/results_2024-02-13 03:33:02.904705.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.700500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-hf (catalogue llama-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b",
    "bench": "triviaqa",
    "score": 0.0949,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-hf/results_2023-12-23%2021%3A45%3A58.973459.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-hf/results_2023-12-23 21:45:58.973459.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.094851; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-hf (catalogue llama-2-13b, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-13b",
    "bench": "natural-questions",
    "score": 0.3222,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-13b-hf/results_2024-02-11%2023%3A31%3A13.502617.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-13b-hf/results_2024-02-11 23:31:13.502617.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.322161; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-13b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-13b-hf (catalogue llama-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "truthfulqa-mc2",
    "score": 0.4531,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-chat-hf/results_2024-02-12%2017%3A46%3A56.163631.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-chat-hf/results_2024-02-12 17:46:56.163631.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.453116; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-chat-hf (catalogue llama-2-7b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "halueval-qa-acc",
    "score": 0.5231,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-chat-hf/results_2023-12-15%2009%3A48%3A47.637297.json",
    "by": "independent",
    "date": "2023-12-15",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-chat-hf/results_2023-12-15 09:48:47.637297.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.523100; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-chat-hf (catalogue llama-2-7b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "triviaqa",
    "score": 0.0883,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-chat-hf/results_2023-12-22%2004%3A04%3A29.898143.json",
    "by": "independent",
    "date": "2023-12-22",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-chat-hf/results_2023-12-22 04:04:29.898143.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.088330; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-chat-hf (catalogue llama-2-7b-chat, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b-chat",
    "bench": "natural-questions",
    "score": 0.2368,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-chat-hf/results_2024-02-12%2002%3A41%3A41.097397.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-chat-hf/results_2024-02-12 02:41:41.097397.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.236842; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-chat-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-chat-hf (catalogue llama-2-7b-chat, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.3897,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-hf/results_2023-12-08%2020%3A10%3A43.772045.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-hf/results_2023-12-08 20:10:43.772045.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.389651; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-hf,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-hf (catalogue llama-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b",
    "bench": "halueval-qa-acc",
    "score": 0.4566,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-hf/results_2023-12-13%2008%3A15%3A43.958836.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-hf/results_2023-12-13 08:15:43.958836.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.456600; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-hf (catalogue llama-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b",
    "bench": "triviaqa",
    "score": 0.0922,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-hf/results_2023-12-22%2004%3A21%3A19.624624.json",
    "by": "independent",
    "date": "2023-12-22",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-hf/results_2023-12-22 04:21:19.624624.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.092176; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-hf (catalogue llama-2-7b, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "llama-2-7b",
    "bench": "natural-questions",
    "score": 0.2712,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/meta-llama/Llama-2-7b-hf/results_2024-02-11%2021%3A35%3A55.239271.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'meta-llama/Llama-2-7b-hf/results_2024-02-11 21:35:55.239271.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.271191; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=meta-llama/Llama-2-7b-hf,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo meta-llama/Llama-2-7b-hf (catalogue llama-2-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "orca-2-13b",
    "bench": "truthfulqa-mc2",
    "score": 0.5576,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/microsoft/Orca-2-13b/results_2023-12-11%2010%3A45%3A23.033484.json",
    "by": "independent",
    "date": "2023-12-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'microsoft/Orca-2-13b/results_2023-12-11 10:45:23.033484.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.557567; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=microsoft/Orca-2-13b,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo microsoft/Orca-2-13b (catalogue orca-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "orca-2-13b",
    "bench": "halueval-qa-acc",
    "score": 0.662,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/microsoft/Orca-2-13b/results_2024-03-12%2017%3A15%3A42.019083.json",
    "by": "independent",
    "date": "2024-03-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'microsoft/Orca-2-13b/results_2024-03-12 17:15:42.019083.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.662000; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=microsoft/Orca-2-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo microsoft/Orca-2-13b (catalogue orca-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "orca-2-13b",
    "bench": "triviaqa",
    "score": 0.0979,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/microsoft/Orca-2-13b/results_2023-12-21%2023%3A45%3A21.730094.json",
    "by": "independent",
    "date": "2023-12-21",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'microsoft/Orca-2-13b/results_2023-12-21 23:45:21.730094.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.097916; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=microsoft/Orca-2-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo microsoft/Orca-2-13b (catalogue orca-2-13b, exact model_repo match). FLAG: far below this model's NQ-Open result and the family's published TriviaQA figures, probably an answer-format problem in this harness run; the file flags nothing and the score is not below the random baseline (0), so it is kept as a valid measurement.",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "orca-2-13b",
    "bench": "natural-questions",
    "score": 0.1778,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/microsoft/Orca-2-13b/results_2024-02-12%2011%3A52%3A32.285173.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'microsoft/Orca-2-13b/results_2024-02-12 11:52:32.285173.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.177839; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=microsoft/Orca-2-13b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo microsoft/Orca-2-13b (catalogue orca-2-13b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "triviaqa",
    "score": 0.5224,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.1/results_2024-02-25%2022%3A42%3A33.912317.json",
    "by": "independent",
    "date": "2024-02-25",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.1/results_2024-02-25 22:42:33.912317.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.522403; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.1 (catalogue mistral-7b-instruct-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "truthfulqa-mc2",
    "score": 0.5592,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.1/results_2023-12-24%2022%3A43%3A28.675599.json",
    "by": "independent",
    "date": "2023-12-24",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.1/results_2023-12-24 22:43:28.675599.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.559211; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.1 (catalogue mistral-7b-instruct-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "halueval-qa-acc",
    "score": 0.4917,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.1/results_2023-12-26%2009%3A44%3A31.720031.json",
    "by": "independent",
    "date": "2023-12-26",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.1/results_2023-12-26 09:44:31.720031.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.491700; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.1 (catalogue mistral-7b-instruct-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-1",
    "bench": "natural-questions",
    "score": 0.1776,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.1/results_2024-02-11%2018%3A31%3A12.754223.json",
    "by": "independent",
    "date": "2024-02-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.1/results_2024-02-11 18:31:12.754223.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.177562; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.1 (catalogue mistral-7b-instruct-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "triviaqa",
    "score": 0.6389,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.2/results_2023-12-24%2016%3A03%3A31.152047.json",
    "by": "independent",
    "date": "2023-12-24",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.2/results_2023-12-24 16:03:31.152047.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.638932; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.2,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.2 (catalogue mistral-7b-instruct-v0-2, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "halueval-qa-acc",
    "score": 0.23,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.2/results_2024-01-02%2006%3A18%3A43.504030.json",
    "by": "independent",
    "date": "2024-01-02",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.2/results_2024-01-02 06:18:43.504030.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.230000; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.2,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.2 (catalogue mistral-7b-instruct-v0-2, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "truthfulqa-mc2",
    "score": 0.6682,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.2/results_2024-01-05%2011%3A31%3A01.352862.json",
    "by": "independent",
    "date": "2024-01-05",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.2/results_2024-01-05 11:31:01.352862.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.668165; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.2,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.2 (catalogue mistral-7b-instruct-v0-2, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-instruct-v0-2",
    "bench": "natural-questions",
    "score": 0.2471,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-Instruct-v0.2/results_2024-02-12%2015%3A18%3A27.775649.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-Instruct-v0.2/results_2024-02-12 15:18:27.775649.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.247091; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-Instruct-v0.2,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-Instruct-v0.2 (catalogue mistral-7b-instruct-v0-2, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "truthfulqa-mc2",
    "score": 0.4263,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-v0.1/results_2023-12-09%2002%3A59%3A23.060797.json",
    "by": "independent",
    "date": "2023-12-09",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-v0.1/results_2023-12-09 02:59:23.060797.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.426294; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-v0.1,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-v0.1 (catalogue mistral-7b-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "halueval-qa-acc",
    "score": 0.5448,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-v0.1/results_2024-02-23%2022%3A54%3A12.386937.json",
    "by": "independent",
    "date": "2024-02-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-v0.1/results_2024-02-23 22:54:12.386937.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.544800; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-v0.1 (catalogue mistral-7b-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "triviaqa",
    "score": 0.7086,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-v0.1/results_2023-12-28%2011%3A09%3A06.178683.json",
    "by": "independent",
    "date": "2023-12-28",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-v0.1/results_2023-12-28 11:09:06.178683.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.708649; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-v0.1 (catalogue mistral-7b-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "mistral-7b-v0-1",
    "bench": "natural-questions",
    "score": 0.3011,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/mistralai/Mistral-7B-v0.1/results_2024-02-12%2009%3A16%3A34.131422.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'mistralai/Mistral-7B-v0.1/results_2024-02-12 09:16:34.131422.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.301108; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=mistralai/Mistral-7B-v0.1,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo mistralai/Mistral-7B-v0.1 (catalogue mistral-7b-v0-1, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.5305,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/teknium/OpenHermes-2.5-Mistral-7B/results_2024-03-03%2012%3A10%3A09.275145.json",
    "by": "independent",
    "date": "2024-03-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'teknium/OpenHermes-2.5-Mistral-7B/results_2024-03-03 12:10:09.275145.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.530496; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=teknium/OpenHermes-2.5-Mistral-7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo teknium/OpenHermes-2.5-Mistral-7B (catalogue openhermes-2-5-mistral-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "halueval-qa-acc",
    "score": 0.5065,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-12%2023%3A31%3A54.586449.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-12 23:31:54.586449.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.506500; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=teknium/OpenHermes-2.5-Mistral-7B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo teknium/OpenHermes-2.5-Mistral-7B (catalogue openhermes-2-5-mistral-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "natural-questions",
    "score": 0.2657,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-12%2015%3A33%3A14.032776.json",
    "by": "independent",
    "date": "2024-02-12",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-12 15:33:14.032776.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.265651; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=teknium/OpenHermes-2.5-Mistral-7B,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo teknium/OpenHermes-2.5-Mistral-7B (catalogue openhermes-2-5-mistral-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "openhermes-2-5-mistral-7b",
    "bench": "triviaqa",
    "score": 0.6598,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-02%2017%3A11%3A11.893825.json",
    "by": "independent",
    "date": "2024-02-02",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'teknium/OpenHermes-2.5-Mistral-7B/results_2024-02-02 17:11:11.893825.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.659831; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=teknium/OpenHermes-2.5-Mistral-7B,revision=main,parallelize=True,dtype=bfloat16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo teknium/OpenHermes-2.5-Mistral-7B (catalogue openhermes-2-5-mistral-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-40b",
    "bench": "natural-questions",
    "score": 0.3197,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-40b/results_2024-01-10%2008%3A05%3A28.458205.json",
    "by": "independent",
    "date": "2024-01-10",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-40b/results_2024-01-10 08:05:28.458205.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.319668; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-40b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-40b (catalogue falcon-40b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-40b",
    "bench": "truthfulqa-mc2",
    "score": 0.4163,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-40b/results_2024-01-11%2018%3A15%3A47.309861.json",
    "by": "independent",
    "date": "2024-01-11",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-40b/results_2024-01-11 18:15:47.309861.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.416296; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-40b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-40b (catalogue falcon-40b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-40b-instruct",
    "bench": "truthfulqa-mc2",
    "score": 0.5247,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-40b-instruct/results_2023-12-25%2018%3A17%3A27.954000.json",
    "by": "independent",
    "date": "2023-12-25",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-40b-instruct/results_2023-12-25 18:17:27.954000.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.524728; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-40b-instruct,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-40b-instruct (catalogue falcon-40b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b",
    "bench": "truthfulqa-mc2",
    "score": 0.3427,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b/results_2024-02-13%2021%3A49%3A09.998773.json",
    "by": "independent",
    "date": "2024-02-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b/results_2024-02-13 21:49:09.998773.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.342653; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b (catalogue falcon-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b",
    "bench": "halueval-qa-acc",
    "score": 0.4672,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b/results_2023-12-16%2002%3A54%3A38.742566.json",
    "by": "independent",
    "date": "2023-12-16",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b/results_2023-12-16 02:54:38.742566.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.467200; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b (catalogue falcon-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b",
    "bench": "triviaqa",
    "score": 0.5867,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b/results_2024-02-14%2018%3A50%3A25.663061.json",
    "by": "independent",
    "date": "2024-02-14",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b/results_2024-02-14 18:50:25.663061.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.586658; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b (catalogue falcon-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b",
    "bench": "natural-questions",
    "score": 0.2208,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b/results_2023-12-29%2010%3A59%3A00.762080.json",
    "by": "independent",
    "date": "2023-12-29",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b/results_2023-12-29 10:59:00.762080.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.220776; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b (catalogue falcon-7b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "truthfulqa-mc2",
    "score": 0.4408,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b-instruct/results_2024-02-17%2018%3A48%3A20.011232.json",
    "by": "independent",
    "date": "2024-02-17",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b-instruct/results_2024-02-17 18:48:20.011232.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.440767; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b-instruct,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b-instruct (catalogue falcon-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "halueval-qa-acc",
    "score": 0.2972,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b-instruct/results_2024-03-03%2005%3A52%3A03.427892.json",
    "by": "independent",
    "date": "2024-03-03",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b-instruct/results_2024-03-03 05:52:03.427892.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.297200; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b-instruct,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b-instruct (catalogue falcon-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "triviaqa",
    "score": 0.3916,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b-instruct/results_2024-02-17%2000%3A50%3A54.701536.json",
    "by": "independent",
    "date": "2024-02-17",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b-instruct/results_2024-02-17 00:50:54.701536.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.391551; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b-instruct,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b-instruct (catalogue falcon-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-7b-instruct",
    "bench": "natural-questions",
    "score": 0.1357,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-7b-instruct/results_2024-01-04%2020%3A46%3A09.405434.json",
    "by": "independent",
    "date": "2024-01-04",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-7b-instruct/results_2024-01-04 20:46:09.405434.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.135734; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-7b-instruct,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-7b-instruct (catalogue falcon-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-rw-1b",
    "bench": "truthfulqa-mc2",
    "score": 0.3597,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-rw-1b/results_2023-12-08%2010%3A44%3A07.732591.json",
    "by": "independent",
    "date": "2023-12-08",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-rw-1b/results_2023-12-08 10:44:07.732591.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.359660; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-rw-1b,revision=main,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-rw-1b (catalogue falcon-rw-1b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-rw-1b",
    "bench": "halueval-qa-acc",
    "score": 0.422,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-rw-1b/results_2023-12-13%2004%3A45%3A23.642544.json",
    "by": "independent",
    "date": "2023-12-13",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-rw-1b/results_2023-12-13 04:45:23.642544.json', HaluEval QA hallucination detection, accuracy (lm-eval task 'halueval_qa'): results['halueval_qa']['acc,none'] = 0.422000; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-rw-1b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-rw-1b (catalogue falcon-rw-1b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-rw-1b",
    "bench": "triviaqa",
    "score": 0.3139,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-rw-1b/results_2023-12-20%2023%3A51%3A12.755266.json",
    "by": "independent",
    "date": "2023-12-20",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-rw-1b/results_2023-12-20 23:51:12.755266.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.313921; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-rw-1b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-rw-1b (catalogue falcon-rw-1b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "falcon-rw-1b",
    "bench": "natural-questions",
    "score": 0.0928,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/tiiuae/falcon-rw-1b/results_2023-12-23%2017%3A09%3A28.422286.json",
    "by": "independent",
    "date": "2023-12-23",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'tiiuae/falcon-rw-1b/results_2023-12-23 17:09:28.422286.json', NQ-Open, 8-shot, exact match (lm-eval task 'nq8'): results['nq8']['exact_match,remove_whitespace'] = 0.092798; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=tiiuae/falcon-rw-1b,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo tiiuae/falcon-rw-1b (catalogue falcon-rw-1b, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "triviaqa",
    "score": 0.4198,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-01-05%2008%3A53%3A29.123401.json",
    "by": "independent",
    "date": "2024-01-05",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-01-05 08:53:29.123401.json', TriviaQA, 8-shot, exact match (lm-eval task 'tqa8'): results['tqa8']['exact_match,remove_whitespace'] = 0.419806; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,revision=main,parallelize=True,dtype=float32'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo upstage/SOLAR-10.7B-Instruct-v1.0 (catalogue solar-10-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "truthfulqa-mc2",
    "score": 0.7172,
    "source": "https://huggingface.co/datasets/hallucinations-leaderboard/results/blob/30f759225f89edcc18cb403b93f27bf4231223f7/upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-02-17%2003%3A25%3A34.987605.json",
    "by": "independent",
    "date": "2024-02-17",
    "note": "Hallucinations Leaderboard results dataset (hallucinations-leaderboard/results @30f7592, Apache-2.0), file 'upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-02-17 03:25:34.987605.json', TruthfulQA MC2, 0-shot (lm-eval task 'truthfulqa_mc2'): results['truthfulqa_mc2']['acc,none'] = 0.717169; read 2026-10-11. An EleutherAI lm-evaluation-harness output written by the leaderboard's own backend (config.model_args 'pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,revision=main,parallelize=True,dtype=float16'). Latest of this model's runs of the task in the dataset (by file timestamp); earlier runs not used. Checkpoint: HF repo upstage/SOLAR-10.7B-Instruct-v1.0 (catalogue solar-10-7b-instruct, exact model_repo match).",
    "source_id": "hallucinations-leaderboard-results-apache"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "medqa-4options",
    "score": 0.5538,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/01-ai/Yi-1.5-9B/results_2024-05-16%2007%3A57%3A23.098609.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file '01-ai/Yi-1.5-9B/results_2024-05-16 07:57:23.098609.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.553810; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=01-ai/Yi-1.5-9B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo 01-ai/Yi-1.5-9B (catalogue yi-1-5-9b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "medmcqa",
    "score": 0.5326,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/01-ai/Yi-1.5-9B/results_2024-05-16%2007%3A57%3A23.098609.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file '01-ai/Yi-1.5-9B/results_2024-05-16 07:57:23.098609.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.532632; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=01-ai/Yi-1.5-9B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo 01-ai/Yi-1.5-9B (catalogue yi-1-5-9b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "yi-1-5-9b-base",
    "bench": "pubmedqa",
    "score": 0.716,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/01-ai/Yi-1.5-9B/results_2024-05-16%2007%3A57%3A23.098609.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file '01-ai/Yi-1.5-9B/results_2024-05-16 07:57:23.098609.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.716000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=01-ai/Yi-1.5-9B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo 01-ai/Yi-1.5-9B (catalogue yi-1-5-9b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1-4b",
    "bench": "medqa-4options",
    "score": 0.2749,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1.4b/results_2024-05-18%2020%3A54%3A29.972980.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1.4b/results_2024-05-18 20:54:29.972980.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.274941; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1.4b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1.4b (catalogue pythia-1-4b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1-4b",
    "bench": "medmcqa",
    "score": 0.3146,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1.4b/results_2024-05-18%2020%3A54%3A29.972980.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1.4b/results_2024-05-18 20:54:29.972980.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.314607; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1.4b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1.4b (catalogue pythia-1-4b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1-4b",
    "bench": "pubmedqa",
    "score": 0.594,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1.4b/results_2024-05-18%2020%3A54%3A29.972980.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1.4b/results_2024-05-18 20:54:29.972980.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.594000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1.4b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1.4b (catalogue pythia-1-4b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1b",
    "bench": "medqa-4options",
    "score": 0.2561,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1b/results_2024-05-20%2011%3A06%3A02.636901.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1b/results_2024-05-20 11:06:02.636901.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.256088; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1b (catalogue pythia-1b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1b",
    "bench": "medmcqa",
    "score": 0.3163,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1b/results_2024-05-20%2011%3A06%3A02.636901.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1b/results_2024-05-20 11:06:02.636901.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.316280; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1b (catalogue pythia-1b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-1b",
    "bench": "pubmedqa",
    "score": 0.582,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-1b/results_2024-05-20%2011%3A06%3A02.636901.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-1b/results_2024-05-20 11:06:02.636901.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.582000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-1b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-1b (catalogue pythia-1b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b",
    "bench": "medqa-4options",
    "score": 0.2435,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b/results_2024-05-20%2011%3A14%3A20.231541.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b/results_2024-05-20 11:14:20.231541.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.243519; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b (catalogue pythia-2-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b",
    "bench": "medmcqa",
    "score": 0.2716,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b/results_2024-05-20%2011%3A14%3A20.231541.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b/results_2024-05-20 11:14:20.231541.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.271575; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b (catalogue pythia-2-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b",
    "bench": "pubmedqa",
    "score": 0.64,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b/results_2024-05-20%2011%3A14%3A20.231541.json",
    "by": "independent",
    "date": "2024-05-20",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b/results_2024-05-20 11:14:20.231541.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.640000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b (catalogue pythia-2-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "medqa-4options",
    "score": 0.2773,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b-deduped/results_2024-05-18%2020%3A42%3A45.067839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b-deduped/results_2024-05-18 20:42:45.067839.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.277298; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b-deduped,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b-deduped (catalogue pythia-2-8b-deduped, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "medmcqa",
    "score": 0.2634,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b-deduped/results_2024-05-18%2020%3A42%3A45.067839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b-deduped/results_2024-05-18 20:42:45.067839.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.263447; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b-deduped,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b-deduped (catalogue pythia-2-8b-deduped, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "pythia-2-8b-deduped",
    "bench": "pubmedqa",
    "score": 0.648,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/EleutherAI/pythia-2.8b-deduped/results_2024-05-18%2020%3A42%3A45.067839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'EleutherAI/pythia-2.8b-deduped/results_2024-05-18 20:42:45.067839.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.648000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=EleutherAI/pythia-2.8b-deduped,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo EleutherAI/pythia-2.8b-deduped (catalogue pythia-2-8b-deduped, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "medqa-4options",
    "score": 0.5247,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Nexusflow/Starling-LM-7B-beta/results_2024-03-31%2023%3A41%3A55.096049.json",
    "by": "independent",
    "date": "2024-03-31",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Nexusflow/Starling-LM-7B-beta/results_2024-03-31 23:41:55.096049.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.524745; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Nexusflow/Starling-LM-7B-beta,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Nexusflow/Starling-LM-7B-beta (catalogue starling-lm-7b-beta, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "medmcqa",
    "score": 0.5025,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Nexusflow/Starling-LM-7B-beta/results_2024-03-31%2023%3A41%3A55.096049.json",
    "by": "independent",
    "date": "2024-03-31",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Nexusflow/Starling-LM-7B-beta/results_2024-03-31 23:41:55.096049.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.502510; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Nexusflow/Starling-LM-7B-beta,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Nexusflow/Starling-LM-7B-beta (catalogue starling-lm-7b-beta, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "starling-lm-7b-beta",
    "bench": "pubmedqa",
    "score": 0.782,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Nexusflow/Starling-LM-7B-beta/results_2024-03-31%2023%3A41%3A55.096049.json",
    "by": "independent",
    "date": "2024-03-31",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Nexusflow/Starling-LM-7B-beta/results_2024-03-31 23:41:55.096049.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.782000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Nexusflow/Starling-LM-7B-beta,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Nexusflow/Starling-LM-7B-beta (catalogue starling-lm-7b-beta, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "nous-hermes-2-mistral-7b-dpo",
    "bench": "medqa-4options",
    "score": 0.531,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01%2000%3A25%3A17.418543.json",
    "by": "independent",
    "date": "2024-04-01",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01 00:25:17.418543.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.531029; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=NousResearch/Nous-Hermes-2-Mistral-7B-DPO,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo NousResearch/Nous-Hermes-2-Mistral-7B-DPO (catalogue nous-hermes-2-mistral-7b-dpo, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "nous-hermes-2-mistral-7b-dpo",
    "bench": "medmcqa",
    "score": 0.4901,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01%2000%3A25%3A17.418543.json",
    "by": "independent",
    "date": "2024-04-01",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01 00:25:17.418543.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.490079; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=NousResearch/Nous-Hermes-2-Mistral-7B-DPO,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo NousResearch/Nous-Hermes-2-Mistral-7B-DPO (catalogue nous-hermes-2-mistral-7b-dpo, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "nous-hermes-2-mistral-7b-dpo",
    "bench": "pubmedqa",
    "score": 0.748,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01%2000%3A25%3A17.418543.json",
    "by": "independent",
    "date": "2024-04-01",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'NousResearch/Nous-Hermes-2-Mistral-7B-DPO/results_2024-04-01 00:25:17.418543.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.748000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=NousResearch/Nous-Hermes-2-Mistral-7B-DPO,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo NousResearch/Nous-Hermes-2-Mistral-7B-DPO (catalogue nous-hermes-2-mistral-7b-dpo, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "medqa-4options",
    "score": 0.454,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Qwen/Qwen1.5-7B-Chat/results_2024-05-16%2015%3A11%3A42.235659.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Qwen/Qwen1.5-7B-Chat/results_2024-05-16 15:11:42.235659.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.454046; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Qwen/Qwen1.5-7B-Chat,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Qwen/Qwen1.5-7B-Chat (catalogue qwen1-5-7b-chat, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "medmcqa",
    "score": 0.4528,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Qwen/Qwen1.5-7B-Chat/results_2024-05-16%2015%3A11%3A42.235659.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Qwen/Qwen1.5-7B-Chat/results_2024-05-16 15:11:42.235659.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.452785; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Qwen/Qwen1.5-7B-Chat,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Qwen/Qwen1.5-7B-Chat (catalogue qwen1-5-7b-chat, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "qwen1-5-7b-chat",
    "bench": "pubmedqa",
    "score": 0.586,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/Qwen/Qwen1.5-7B-Chat/results_2024-05-16%2015%3A11%3A42.235659.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'Qwen/Qwen1.5-7B-Chat/results_2024-05-16 15:11:42.235659.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.586000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=Qwen/Qwen1.5-7B-Chat,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo Qwen/Qwen1.5-7B-Chat (catalogue qwen1-5-7b-chat, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama3-openbiollm-8b",
    "bench": "medqa-4options",
    "score": 0.5782,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29%2009%3A08%3A41.270228.json",
    "by": "independent",
    "date": "2024-05-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29 09:08:41.270228.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.578162; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=aaditya/Llama3-OpenBioLLM-8B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo aaditya/Llama3-OpenBioLLM-8B (catalogue llama3-openbiollm-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama3-openbiollm-8b",
    "bench": "medmcqa",
    "score": 0.5776,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29%2009%3A08%3A41.270228.json",
    "by": "independent",
    "date": "2024-05-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29 09:08:41.270228.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.577576; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=aaditya/Llama3-OpenBioLLM-8B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo aaditya/Llama3-OpenBioLLM-8B (catalogue llama3-openbiollm-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama3-openbiollm-8b",
    "bench": "pubmedqa",
    "score": 0.72,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29%2009%3A08%3A41.270228.json",
    "by": "independent",
    "date": "2024-05-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'aaditya/Llama3-OpenBioLLM-8B/results_2024-05-29 09:08:41.270228.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.720000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=aaditya/Llama3-OpenBioLLM-8B,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo aaditya/Llama3-OpenBioLLM-8B (catalogue llama3-openbiollm-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "ghost-7b-alpha",
    "bench": "medqa-4options",
    "score": 0.403,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/ghost-x/ghost-7b-alpha/results_2024-04-23%2007%3A04%3A42.080410.json",
    "by": "independent",
    "date": "2024-04-23",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'ghost-x/ghost-7b-alpha/results_2024-04-23 07:04:42.080410.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.402985; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=ghost-x/ghost-7b-alpha,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo ghost-x/ghost-7b-alpha (catalogue ghost-7b-alpha, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "ghost-7b-alpha",
    "bench": "medmcqa",
    "score": 0.3806,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/ghost-x/ghost-7b-alpha/results_2024-04-23%2007%3A04%3A42.080410.json",
    "by": "independent",
    "date": "2024-04-23",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'ghost-x/ghost-7b-alpha/results_2024-04-23 07:04:42.080410.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.380588; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=ghost-x/ghost-7b-alpha,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo ghost-x/ghost-7b-alpha (catalogue ghost-7b-alpha, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "ghost-7b-alpha",
    "bench": "pubmedqa",
    "score": 0.692,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/ghost-x/ghost-7b-alpha/results_2024-04-23%2007%3A04%3A42.080410.json",
    "by": "independent",
    "date": "2024-04-23",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'ghost-x/ghost-7b-alpha/results_2024-04-23 07:04:42.080410.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.692000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=ghost-x/ghost-7b-alpha,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo ghost-x/ghost-7b-alpha (catalogue ghost-7b-alpha, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "medqa-4options",
    "score": 0.4572,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-1.1-7b-it/results_2024-05-16%2018%3A07%3A15.583932.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-1.1-7b-it/results_2024-05-16 18:07:15.583932.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.457188; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-1.1-7b-it,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-1.1-7b-it (catalogue gemma-1-1-7b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "medmcqa",
    "score": 0.4439,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-1.1-7b-it/results_2024-05-16%2018%3A07%3A15.583932.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-1.1-7b-it/results_2024-05-16 18:07:15.583932.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.443940; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-1.1-7b-it,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-1.1-7b-it (catalogue gemma-1-1-7b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-1-1-7b",
    "bench": "pubmedqa",
    "score": 0.708,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-1.1-7b-it/results_2024-05-16%2018%3A07%3A15.583932.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-1.1-7b-it/results_2024-05-16 18:07:15.583932.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.708000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-1.1-7b-it,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-1.1-7b-it (catalogue gemma-1-1-7b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "medqa-4options",
    "score": 0.2922,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-2b/results_2024-05-18%2022%3A12%3A37.308839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-2b/results_2024-05-18 22:12:37.308839.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.292223; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-2b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "medmcqa",
    "score": 0.3022,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-2b/results_2024-05-18%2022%3A12%3A37.308839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-2b/results_2024-05-18 22:12:37.308839.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.302175; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-2b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-2b-base",
    "bench": "pubmedqa",
    "score": 0.664,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-2b/results_2024-05-18%2022%3A12%3A37.308839.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-2b/results_2024-05-18 22:12:37.308839.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.664000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-2b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-2b (catalogue gemma-2b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-7b-base",
    "bench": "medqa-4options",
    "score": 0.487,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-7b/results_2024-05-16%2018%3A02%3A08.310603.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-7b/results_2024-05-16 18:02:08.310603.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.487038; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-7b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-7b (catalogue gemma-7b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-7b-base",
    "bench": "medmcqa",
    "score": 0.4934,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-7b/results_2024-05-16%2018%3A02%3A08.310603.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-7b/results_2024-05-16 18:02:08.310603.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.493426; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-7b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-7b (catalogue gemma-7b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gemma-7b-base",
    "bench": "pubmedqa",
    "score": 0.756,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/google/gemma-7b/results_2024-05-16%2018%3A02%3A08.310603.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'google/gemma-7b/results_2024-05-16 18:02:08.310603.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.756000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=google/gemma-7b,revision=main,dtype=bfloat16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo google/gemma-7b (catalogue gemma-7b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "medqa-4options",
    "score": 0.597,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B/results_2024-04-30%2002%3A10%3A59.556091.json",
    "by": "independent",
    "date": "2024-04-30",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B/results_2024-04-30 02:10:59.556091.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.597015; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B (catalogue llama-3-8b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "medmcqa",
    "score": 0.5747,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B/results_2024-04-30%2002%3A10%3A59.556091.json",
    "by": "independent",
    "date": "2024-04-30",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B/results_2024-04-30 02:10:59.556091.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.574707; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B (catalogue llama-3-8b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b-base",
    "bench": "pubmedqa",
    "score": 0.748,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B/results_2024-04-30%2002%3A10%3A59.556091.json",
    "by": "independent",
    "date": "2024-04-30",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B/results_2024-04-30 02:10:59.556091.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.748000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B (catalogue llama-3-8b-base, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "medqa-4options",
    "score": 0.6072,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22%2022%3A18%3A47.513878.json",
    "by": "independent",
    "date": "2024-04-22",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22 22:18:47.513878.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.607227; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B-Instruct,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B-Instruct (catalogue llama-3-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "medmcqa",
    "score": 0.5685,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22%2022%3A18%3A47.513878.json",
    "by": "independent",
    "date": "2024-04-22",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22 22:18:47.513878.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.568492; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B-Instruct,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B-Instruct (catalogue llama-3-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "llama-3-8b",
    "bench": "pubmedqa",
    "score": 0.746,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22%2022%3A18%3A47.513878.json",
    "by": "independent",
    "date": "2024-04-22",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'meta-llama/Meta-Llama-3-8B-Instruct/results_2024-04-22 22:18:47.513878.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.746000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=meta-llama/Meta-Llama-3-8B-Instruct,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo meta-llama/Meta-Llama-3-8B-Instruct (catalogue llama-3-8b, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "phi-1-5",
    "bench": "medqa-4options",
    "score": 0.3111,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/microsoft/phi-1_5/results_2024-05-18%2020%3A59%3A28.331099.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'microsoft/phi-1_5/results_2024-05-18 20:59:28.331099.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.311076; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=microsoft/phi-1_5,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo microsoft/phi-1_5 (catalogue phi-1-5, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "phi-1-5",
    "bench": "medmcqa",
    "score": 0.3431,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/microsoft/phi-1_5/results_2024-05-18%2020%3A59%3A28.331099.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'microsoft/phi-1_5/results_2024-05-18 20:59:28.331099.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.343055; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=microsoft/phi-1_5,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo microsoft/phi-1_5 (catalogue phi-1-5, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "phi-1-5",
    "bench": "pubmedqa",
    "score": 0.678,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/microsoft/phi-1_5/results_2024-05-18%2020%3A59%3A28.331099.json",
    "by": "independent",
    "date": "2024-05-18",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'microsoft/phi-1_5/results_2024-05-18 20:59:28.331099.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.678000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=microsoft/phi-1_5,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo microsoft/phi-1_5 (catalogue phi-1-5, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gpt-2-xl",
    "bench": "medqa-4options",
    "score": 0.2804,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/openai-community/gpt2-xl/results_2024-04-29%2013%3A09%3A52.222616.json",
    "by": "independent",
    "date": "2024-04-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'openai-community/gpt2-xl/results_2024-04-29 13:09:52.222616.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.280440; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=openai-community/gpt2-xl,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo openai-community/gpt2-xl (catalogue gpt-2-xl, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gpt-2-xl",
    "bench": "medmcqa",
    "score": 0.3086,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/openai-community/gpt2-xl/results_2024-04-29%2013%3A09%3A52.222616.json",
    "by": "independent",
    "date": "2024-04-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'openai-community/gpt2-xl/results_2024-04-29 13:09:52.222616.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.308630; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=openai-community/gpt2-xl,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo openai-community/gpt2-xl (catalogue gpt-2-xl, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "gpt-2-xl",
    "bench": "pubmedqa",
    "score": 0.604,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/openai-community/gpt2-xl/results_2024-04-29%2013%3A09%3A52.222616.json",
    "by": "independent",
    "date": "2024-04-29",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'openai-community/gpt2-xl/results_2024-04-29 13:09:52.222616.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.604000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=openai-community/gpt2-xl,revision=main,dtype=float32'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo openai-community/gpt2-xl (catalogue gpt-2-xl, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "medqa-4options",
    "score": 0.5467,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16%2015%3A38%3A23.436788.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16 15:38:23.436788.json', MedQA (USMLE, 4 options) test, accuracy, lm-eval task 'medqa_4options': results['medqa_4options']['acc,none'] = 0.546740; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo upstage/SOLAR-10.7B-Instruct-v1.0 (catalogue solar-10-7b-instruct, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "medmcqa",
    "score": 0.4745,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16%2015%3A38%3A23.436788.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16 15:38:23.436788.json', MedMCQA validation, accuracy, lm-eval task 'medmcqa': results['medmcqa']['acc,none'] = 0.474540; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo upstage/SOLAR-10.7B-Instruct-v1.0 (catalogue solar-10-7b-instruct, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  },
  {
    "model": "solar-10-7b-instruct",
    "bench": "pubmedqa",
    "score": 0.526,
    "source": "https://huggingface.co/datasets/openlifescienceai/results/blob/62d7322cf11bf622399f7f54e5a7620ec6ef013d/upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16%2015%3A38%3A23.436788.json",
    "by": "independent",
    "date": "2024-05-16",
    "note": "Open Medical-LLM Leaderboard results (openlifescienceai/results @62d7322, Apache-2.0), file 'upstage/SOLAR-10.7B-Instruct-v1.0/results_2024-05-16 15:38:23.436788.json', PubMedQA labelled set, accuracy, lm-eval task 'pubmedqa': results['pubmedqa']['acc,none'] = 0.526000; read 2026-10-11. An lm-evaluation-harness output from the leaderboard backend (config.model_args 'pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,revision=main,dtype=float16'); zero-shot as in the harness task. Only harness-generated files are used (hand-entered rows such as Flan-PaLM, GPT-4, Med-PaLM 2, and files without harness metadata are not). Checkpoint: HF repo upstage/SOLAR-10.7B-Instruct-v1.0 (catalogue solar-10-7b-instruct, exact model_repo match).",
    "source_id": "openlifescienceai-results-apache"
  }
]